From ac9d475697f0e6bf0dcf9ab273d7ad7507218ff6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:01:32 +0000 Subject: [PATCH] refactor(typing): narrow Any sources across providers, proxy and guardrails (wave 1) --- enterprise/enterprise_hooks/aporia_ai.py | 4 +- .../example_logging_api.py | 7 +- .../proxy/hooks/managed_files.py | 2 +- litellm/__init__.py | 16 +++- litellm/_lazy_imports.py | 4 +- .../litellm_completion_bridge/handler.py | 2 +- litellm/a2a_protocol/utils.py | 2 +- .../exceptions/exception_mapping_utils.py | 11 ++- litellm/assistants/main.py | 24 ++--- litellm/caching/azure_blob_cache.py | 8 +- litellm/caching/disk_cache.py | 6 +- litellm/cost_calculator.py | 5 +- litellm/exceptions.py | 2 +- litellm/files/main.py | 14 ++- litellm/harness/runtime.py | 8 +- litellm/images/main.py | 11 ++- litellm/integrations/arize/arize.py | 2 +- litellm/integrations/custom_guardrail.py | 2 +- .../focus/destinations/factory.py | 5 +- litellm/integrations/focus/export_engine.py | 8 +- litellm/integrations/focus/focus_logger.py | 8 +- .../gcs_bucket/gcs_bucket_mock_client.py | 3 +- litellm/integrations/gcs_pubsub/pub_sub.py | 8 +- litellm/integrations/otel/model/metadata.py | 6 +- litellm/integrations/otel/plumbing/metrics.py | 8 +- litellm/litellm_core_utils/core_helpers.py | 19 ++-- litellm/litellm_core_utils/cost_discount.py | 2 +- .../litellm_core_utils/default_encoding.py | 2 +- .../dot_notation_indexing.py | 2 +- .../litellm_core_utils/get_model_cost_map.py | 6 +- litellm/litellm_core_utils/litellm_logging.py | 42 +++++---- litellm/litellm_core_utils/llm_judge.py | 5 +- .../logging_callback_manager.py | 8 +- .../prompt_templates/common_utils.py | 2 +- .../prompt_templates/factory.py | 6 +- .../sensitive_data_masker.py | 8 +- .../litellm_core_utils/streaming_handler.py | 7 +- litellm/llms/anthropic/batches/handler.py | 2 +- .../chat/guardrail_translation/handler.py | 9 +- .../editors/clear_tool_uses.py | 15 ++- .../context_management/placeholders.py | 4 +- .../pass_through/context_management/result.py | 6 +- .../messages/fake_stream_iterator.py | 12 +-- .../responses_adapters/transformation.py | 6 +- litellm/llms/azure/audio_transcriptions.py | 4 +- litellm/llms/azure/azure.py | 8 +- .../llms/azure/responses/transformation.py | 2 +- litellm/llms/azure_ai/chat/transformation.py | 2 +- litellm/llms/base_llm/chat/transformation.py | 4 +- .../guardrail_translation/base_translation.py | 2 +- .../base_managed_resource.py | 6 +- .../base_llm/vector_store/transformation.py | 4 +- .../bedrock/chat/agentcore/transformation.py | 15 +-- .../bedrock/chat/mantle/transformation.py | 5 +- litellm/llms/bedrock/files/handler.py | 4 +- .../amazon_titan_transformation.py | 6 +- .../guardrail_translation/handler.py | 6 +- .../image_edit/transformation.py | 3 +- litellm/llms/chatgpt/chat/transformation.py | 2 +- litellm/llms/cometapi/chat/transformation.py | 4 +- .../llms/compactifai/chat/transformation.py | 43 ++++++--- litellm/llms/custom_httpx/llm_http_handler.py | 37 ++++++-- litellm/llms/custom_httpx/mock_transport.py | 13 ++- .../llms/databricks/embed/transformation.py | 6 +- .../deepagents/harness/sandbox_backend.py | 4 +- .../llms/deepagents/harness/transformation.py | 7 +- .../audio_transcription/transformation.py | 42 ++++++--- .../llms/deepseek/messages/transformation.py | 36 +++++--- litellm/llms/deprecated_providers/palm.py | 6 +- .../llms/edenai/embedding/transformation.py | 19 +++- .../edenai/image_generation/transformation.py | 19 +++- .../audio_transcription/transformation.py | 35 ++++--- litellm/llms/gemini/agents/transformation.py | 55 +++++++---- litellm/llms/gemini/common_utils.py | 15 +-- .../gemini/google_genai/transformation.py | 17 +++- .../llms/gemini/image_edit/transformation.py | 43 ++++++--- .../responses/transformation.py | 28 ++++-- litellm/llms/groq/cost_calculator.py | 5 +- litellm/llms/groq/stt/transformation.py | 6 +- .../llms/huggingface/rerank/transformation.py | 2 +- litellm/llms/langgraph/chat/sse_iterator.py | 5 +- litellm/llms/lemonade/chat/transformation.py | 7 +- .../litellm_proxy/skills/sandbox_executor.py | 12 ++- .../llms/lm_studio/embed/transformation.py | 6 +- litellm/llms/nvidia_nim/embed.py | 6 +- litellm/llms/oci/chat/transformation.py | 3 +- litellm/llms/ollama/common_utils.py | 3 +- litellm/llms/openai/common_utils.py | 15 ++- .../openai/responses/count_tokens/handler.py | 7 +- .../responses/count_tokens/token_counter.py | 4 +- .../openai_like/messages/transformation.py | 6 +- .../opensandbox/sandbox/transformation.py | 12 ++- .../perplexity/responses/transformation.py | 5 +- .../image_generation/transformation.py | 17 ++-- .../runwayml/text_to_speech/transformation.py | 17 ++-- .../llms/runwayml/videos/transformation.py | 2 +- litellm/llms/sagemaker/common_utils.py | 11 ++- .../embedding/cohere_transformation.py | 6 +- .../sagemaker/embedding/transformation.py | 6 +- litellm/llms/sap/chat/transformation.py | 8 +- litellm/llms/sap/credentials.py | 27 +++++- litellm/llms/sap/embed/transformation.py | 8 +- .../stability/image_edit/transformations.py | 11 ++- .../llms/triton/completion/transformation.py | 7 +- .../llms/triton/embedding/transformation.py | 16 +++- .../vercel_ai_gateway/chat/transformation.py | 14 ++- .../llms/vertex_ai/batches/transformation.py | 2 +- litellm/llms/vertex_ai/files/handler.py | 4 +- .../llms/vertex_ai/files/transformation.py | 14 ++- litellm/llms/vertex_ai/fine_tuning/handler.py | 13 +-- litellm/llms/vertex_ai/gemini/long_decode.py | 3 +- .../batch_embed_content_handler.py | 10 +- .../vertex_imagen_transformation.py | 7 +- .../image_generation_handler.py | 4 +- .../multimodal_embeddings/transformation.py | 2 +- .../vertex_ai/rag_engine/transformation.py | 18 +++- .../llms/vertex_ai/rerank/transformation.py | 16 +++- .../ai21/transformation.py | 6 +- .../vertex_embeddings/transformation.py | 6 +- .../volcengine/responses/transformation.py | 11 ++- litellm/main.py | 12 +-- litellm/proxy/_experimental/mcp_server/db.py | 3 +- .../mcp_server/discoverable_endpoints.py | 2 +- .../mcp_server/mcp_server_manager.py | 8 +- .../mcp_server/oauth2_token_cache.py | 5 +- .../mcp_server/oauth_identity_binding.py | 6 +- .../mcp_server/result_conversion.py | 13 ++- .../mcp_server/semantic_tool_filter.py | 14 +-- .../proxy/_experimental/mcp_server/server.py | 20 +++- .../proxy/_experimental/mcp_server/utils.py | 5 +- litellm/proxy/_lazy_features.py | 20 ++-- litellm/proxy/_lazy_openapi_snapshot.py | 45 +++++---- litellm/proxy/a2a/discovery.py | 14 ++- .../proxy/agent_endpoints/a2a_endpoints.py | 15 +-- litellm/proxy/agent_endpoints/a2a_routing.py | 6 +- .../claude_code_marketplace.py | 3 +- litellm/proxy/auth/auth_utils.py | 16 +++- litellm/proxy/auth/ip_address_utils.py | 4 +- litellm/proxy/auth/route_checks.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 3 +- litellm/proxy/client/chat.py | 5 +- .../proxy/client/cli/commands/encryption.py | 23 +++-- litellm/proxy/client/credentials.py | 6 +- litellm/proxy/client/http_client.py | 7 +- litellm/proxy/client/model_groups.py | 4 +- litellm/proxy/client/models.py | 2 +- .../auth_cache_invalidation_pubsub.py | 15 ++- litellm/proxy/common_utils/callback_utils.py | 18 ++-- .../proxy/common_utils/http_parsing_utils.py | 2 +- .../common_utils/proxy_rate_limit_error.py | 4 +- .../container_endpoints/handler_factory.py | 5 +- litellm/proxy/db/autorouter_session_rollup.py | 2 +- .../db_transaction_queue/pod_lock_manager.py | 6 +- litellm/proxy/db/routing_prisma_wrapper.py | 2 +- .../proxy/guardrails/guardrail_endpoints.py | 3 +- .../guardrail_hooks/agent_365/agent_365.py | 5 +- .../guardrails/guardrail_hooks/airia/airia.py | 13 ++- .../guardrail_hooks/compresr/compresr.py | 4 +- .../generic_guardrail_api.py | 12 ++- .../guardrail_hooks/grayswan/grayswan.py | 5 +- .../guardrail_hooks/javelin/javelin.py | 7 +- .../guardrail_hooks/lakera_ai_v2.py | 7 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 43 +++++---- .../competitor_intent/airline.py | 8 +- .../litellm_content_filter/content_filter.py | 2 +- .../llm_shield_proxy/stream_restorers.py | 6 +- .../mcp_jwt_signer/mcp_jwt_signer.py | 19 ++-- .../mcp_security/mcp_security_guardrail.py | 7 +- .../microsoft_purview/__init__.py | 5 +- .../guardrail_hooks/noma/noma_v2.py | 24 +++-- .../guardrails/guardrail_hooks/onyx/onyx.py | 5 +- .../guardrail_hooks/openai/moderations.py | 7 +- .../panw_prisma_airs/panw_prisma_airs.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 6 +- .../guardrail_hooks/qualifire/qualifire.py | 4 +- .../semantic_guard/route_loader.py | 4 +- .../unified_guardrail/unified_guardrail.py | 5 +- .../proxy/guardrails/tool_name_extraction.py | 8 +- .../proxy/hooks/key_management_event_hooks.py | 2 +- .../proxy/hooks/parallel_request_limiter.py | 2 +- .../hooks/parallel_request_limiter_v3.py | 2 +- .../model_management_endpoints.py | 11 ++- .../management_endpoints/team_endpoints.py | 2 +- litellm/proxy/management_endpoints/ui_sso.py | 2 +- .../object_permission_utils.py | 2 +- .../llm_passthrough_endpoints.py | 2 +- .../tinyfish_passthrough_logging_handler.py | 5 +- .../transcribe_passthrough_logging_handler.py | 9 +- .../upstream_usage_headers.py | 3 +- litellm/proxy/proxy_server.py | 13 ++- .../spend_tracking/ptu_flat_cost_rollup.py | 7 +- litellm/proxy/utils.py | 3 +- litellm/proxy/video_endpoints/endpoints.py | 6 +- litellm/proxy/video_endpoints/utils.py | 2 +- litellm/rag/ingestion/bedrock_ingestion.py | 4 +- .../streaming_iterator.py | 3 +- litellm/responses/streaming_iterator.py | 5 +- litellm/responses/utils.py | 92 ++++++++++--------- litellm/router.py | 2 +- litellm/utils.py | 18 ++-- litellm/vector_stores/main.py | 10 +- .../code_coverage_tests/recursive_detector.py | 2 +- 202 files changed, 1304 insertions(+), 644 deletions(-) diff --git a/enterprise/enterprise_hooks/aporia_ai.py b/enterprise/enterprise_hooks/aporia_ai.py index 28b49bfce21..fbde23286e3 100644 --- a/enterprise/enterprise_hooks/aporia_ai.py +++ b/enterprise/enterprise_hooks/aporia_ai.py @@ -15,7 +15,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import json import sys -from typing import Any, List, Literal, Optional +from typing import List, Literal, Optional from fastapi import HTTPException @@ -68,7 +68,7 @@ class AporiaGuardrail(CustomGuardrail): async def prepare_aporia_request( self, new_messages: List[dict], response_string: Optional[str] = None ) -> dict: - data: dict[str, Any] = {} + data: dict[str, object] = {} if new_messages is not None: data["messages"] = new_messages if response_string is not None: diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py index 14d34f5d1e8..e0190ea8da0 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py @@ -1,15 +1,20 @@ # this is an example endpoint to receive data from litellm +from typing import Final + from fastapi import FastAPI, HTTPException, Request +from pydantic import TypeAdapter app = FastAPI() +_LOGGED_EVENT: Final = TypeAdapter(object) + @app.post("/log-event") async def log_event(request: Request): try: print("Received /log-event request") # noqa # Assuming the incoming request has JSON data - data = await request.json() + data: Final = _LOGGED_EVENT.validate_python(await request.json()) print("Received request data:") # noqa print(data) # noqa diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bc35c4a0aba..6eff5411e70 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -2652,7 +2652,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Continue with other files even if one fails continue - def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str: + def _get_content_type_from_file_object(self, file_object: object) -> str: """ Determine content type from file object. diff --git a/litellm/__init__.py b/litellm/__init__.py index 0fe3d091c2a..a50ba3622e6 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload -argilla_transformation_object: Optional[Dict[str, Any]] = None +argilla_transformation_object: Mapping[str, str] | None = None _async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. @@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False # Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy -proxy_auth: Optional[Any] = None +proxy_auth: _ProxyAuthHandlerType | None = None ### DEFAULT AZURE API VERSION ### AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest ### DEFAULT WATSONX API VERSION ### @@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) -public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} +public_model_groups_links: dict[str, str | dict[str, object]] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None # priority_reservation_settings is lazy-loaded via __getattr__ @@ -2284,6 +2284,10 @@ if TYPE_CHECKING: LiteLLM_UpperboundKeyGenerateParams, ) from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator + from collections.abc import Awaitable + from litellm.types.utils import ModelResponse as _ModelResponseType + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType + from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -2307,7 +2311,7 @@ if TYPE_CHECKING: supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] - acreate: Callable[..., Any] + acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]] get_max_tokens: Callable[..., int] get_model_info: Callable[..., _ModelInfoType] register_prompt_template: Callable[..., None] @@ -2605,8 +2609,10 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) + from types import MappingProxyType - return locals()[name] + evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals()) + return evals_exports[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 46126c444a6..1a7dc343745 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -19,7 +19,7 @@ import importlib import sys from collections.abc import Callable, Mapping from types import ModuleType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final from typing_extensions import ReadOnly, TypedDict @@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object: params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"} # Create the client instance - provider_id: Final = cast(Any, "litellm_module_level_client") + provider_id: Final = "litellm_module_level_client" async_client: Final = get_async_httpx_client( llm_provider=provider_id, params=params, diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 0002a597d6a..6d3b09aed68 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler: @staticmethod def _build_completion_params( params: dict[str, Any], - litellm_params: Mapping[str, Any], + litellm_params: Mapping[str, object], api_base: str | None, agent_extra_headers: Mapping[str, str] | None, *, diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index 7400844bf28..460f4eade0f 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -84,7 +84,7 @@ class A2ARequestUtils: Returns: The message object/dict or None """ - params: Final = getattr(request, "params", None) + params: Final[object] = getattr(request, "params", None) if params is None: return None return getattr(params, "message", None) diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index eb3ec8aaee2..d8198e41239 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop import json from typing import Final +from pydantic import TypeAdapter + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType +_PARSED_ERROR_BODY: Final = TypeAdapter(object) + # HTTP status code -> Anthropic error type # Source: https://docs.anthropic.com/en/api/errors ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { @@ -79,7 +83,7 @@ class AnthropicExceptionMapping: - Generic: {"message": "..."} - Plain strings """ - parsed: Final = safe_json_loads(raw_message) + parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) if isinstance(parsed, dict): # Bedrock format if "detail" in parsed and isinstance(parsed["detail"], dict): @@ -145,9 +149,8 @@ class AnthropicExceptionMapping: AnthropicErrorResponse dict """ # Try to parse as JSON once - parsed: dict | None = safe_json_loads(raw_message) - if not isinstance(parsed, dict): - parsed = None + decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) + parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None # If parsed and already in Anthropic format - passthrough if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed): diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index a3428261636..a5721c6f047 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -77,7 +77,7 @@ def get_assistants( api_base: str | None = None, api_version: str | None = None, **kwargs, -) -> SyncCursorPage[Assistant]: +) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]: aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None) if aget_assistants is not None and not isinstance(aget_assistants, bool): raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed") @@ -100,7 +100,7 @@ def get_assistants( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[Assistant] | None = None + response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -564,7 +564,7 @@ def create_thread( tool_resources: OpenAICreateThreadParamsToolResources | None = None, client: OpenAI | None = None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """ - get the llm provider - if openai - route it there @@ -610,7 +610,7 @@ def create_thread( api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -735,7 +735,7 @@ def get_thread( thread_id: str, client=None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """Get the thread object, given a thread_id""" aget_thread: Final = kwargs.pop("aget_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -757,7 +757,7 @@ def get_thread( timeout = 600.0 api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -903,7 +903,7 @@ def add_message( metadata: dict | None = None, client=None, **kwargs, -) -> OpenAIMessage: +) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]: ### COMMON OBJECTS ### a_add_message: Final = kwargs.pop("a_add_message", None) _message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata) @@ -935,7 +935,7 @@ def add_message( timeout = 600.0 api_key: str | None = None api_base: str | None = None - response: OpenAIMessage | None = None + response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -1065,7 +1065,7 @@ def get_messages( thread_id: str, client: Any | None = None, **kwargs, -) -> SyncCursorPage[OpenAIMessage]: +) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: aget_messages: Final = kwargs.pop("aget_messages", None) optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) @@ -1086,7 +1086,7 @@ def get_messages( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[OpenAIMessage] | None = None + response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None api_key: str | None = None api_base: str | None = None if custom_llm_provider == "openai": @@ -1256,7 +1256,7 @@ def run_thread( client: object | None = None, event_handler: AssistantEventHandler | None = None, # for stream=True calls **kwargs, -) -> Run: +) -> Run | Coroutine[None, None, Run]: """Run a given thread + assistant.""" arun_thread: Final = kwargs.pop("arun_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -1278,7 +1278,7 @@ def run_thread( elif timeout is None: timeout = 600.0 - response: Run | None = None + response: Run | Coroutine[None, None, Run] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there diff --git a/litellm/caching/azure_blob_cache.py b/litellm/caching/azure_blob_cache.py index bf0e15421bc..5b9071dc920 100644 --- a/litellm/caching/azure_blob_cache.py +++ b/litellm/caching/azure_blob_cache.py @@ -13,10 +13,14 @@ import json from contextlib import suppress from typing import Final +from pydantic import TypeAdapter + from litellm._logging import print_verbose, verbose_logger from .base_cache import BaseCache +_DECODED_BLOB: Final = TypeAdapter(object) + class AzureBlobCache(BaseCache): def __init__(self, account_url, container) -> None: @@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache): print_verbose(f"Get Azure Blob Cache: key: {key}") as_bytes: Final = self.container_client.download_blob(key).readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", @@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache): blob: Final = await self.async_container_client.download_blob(key) as_bytes: Final = await blob.readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", key, diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py index 50939ad51ca..6d74faf5016 100644 --- a/litellm/caching/disk_cache.py +++ b/litellm/caching/disk_cache.py @@ -1,6 +1,8 @@ import json from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from .base_cache import BaseCache if TYPE_CHECKING: @@ -10,6 +12,8 @@ if TYPE_CHECKING: else: Span = Any +_DECODED_DISK_VALUE: Final = TypeAdapter(object) + class DiskCache(BaseCache): def __init__(self, disk_cache_dir: str | None = None): @@ -44,7 +48,7 @@ class DiskCache(BaseCache): original_cached_response: Final = self.disk_cache.get(key) if original_cached_response: try: - cached_response = json.loads(original_cached_response) + cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response)) except Exception: cached_response = original_cached_response return cached_response diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 20914df8fe2..15c93e274d2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2994,8 +2994,11 @@ def _batch_prompt_cost( ) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) + + def _attribute_value(obj: object, name: str) -> object: - return getattr(obj, name) + return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name)) def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]: diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 733f1aa72d5..b8e933bf046 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError): self.message = _saved_message self.args = (_saved_message,) - original_headers: Final = getattr(original_exception, "litellm_response_headers", None) + original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None) if original_headers is not None: self.litellm_response_headers = original_headers diff --git a/litellm/files/main.py b/litellm/files/main.py index 16ff5b02f9c..c5172bc01ae 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -9,7 +9,7 @@ import asyncio import contextvars import time import uuid as uuid_module -from collections.abc import Coroutine +from collections.abc import Awaitable, Coroutine from functools import partial from typing import Final, Literal, cast @@ -293,6 +293,14 @@ def create_file( raise e +async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject: + return await pending + + +async def _await_deleted_file(pending: Awaitable[object]) -> object: + return await pending + + @client async def afile_retrieve( file_id: str, @@ -325,7 +333,7 @@ async def afile_retrieve( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_retrieved_file(init_response) else: response = init_response @@ -505,7 +513,7 @@ async def afile_delete( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_deleted_file(init_response) else: response = init_response diff --git a/litellm/harness/runtime.py b/litellm/harness/runtime.py index dc24a1f0ef7..4b95805a940 100644 --- a/litellm/harness/runtime.py +++ b/litellm/harness/runtime.py @@ -12,7 +12,7 @@ import inspect import logging import os import uuid -from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType from typing import ( @@ -331,11 +331,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None: queue.put_nowait(end) +async def _await_decision(pending: Awaitable[object]) -> object: + return await pending + + async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None: """Run on_approval (sync in a worker thread, or async) and resolve approval.""" try: if inspect.iscoroutinefunction(handler): - decision: object = await handler(approval) + decision: object = await _await_decision(handler(approval)) else: decision = await asyncio.to_thread(handler, approval) if inspect.isawaitable(decision): diff --git a/litellm/images/main.py b/litellm/images/main.py index 7f94835d59b..29bac3ffe05 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -3,7 +3,7 @@ import contextvars import importlib from collections.abc import Coroutine from functools import partial -from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, overload from pydantic import TypeAdapter, ValidationError @@ -70,16 +70,17 @@ from litellm.utils import ( _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) # Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls -_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None +_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None -def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils": +def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]: """Get ImageEditRequestUtils, loading it lazily if needed.""" global _ImageEditRequestUtils_cache if _ImageEditRequestUtils_cache is None: # Access via module to trigger __getattr__ if not cached - module: Final = importlib.import_module(__name__) - _ImageEditRequestUtils_cache = module.ImageEditRequestUtils + from litellm.images import main as images_main + + _ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker return _ImageEditRequestUtils_cache diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 24279b443aa..5a03b91d470 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry): primary ``otel`` callback which handles proxy-level parent spans. """ - def set_attributes(self, span: Span, kwargs, response_obj: Any | None): + def set_attributes(self, span: Span, kwargs, response_obj: object): ArizeLogger.set_arize_attributes(span, kwargs, response_obj) @staticmethod diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 1c12ac395c7..85bd27adc86 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger): self, violation_message: str, request_data: dict[str, Any], - detection_info: dict[str, Any] | None = None, + detection_info: Mapping[str, object] | None = None, original_response: object = None, ) -> None: """ diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py index 0ecf04326b3..cf871013265 100644 --- a/litellm/integrations/focus/destinations/factory.py +++ b/litellm/integrations/focus/destinations/factory.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from typing import Any, Final from .base import FocusDestination @@ -20,7 +21,7 @@ class FocusDestinationFactory: *, provider: str, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, object] | None = None, ) -> FocusDestination: """Return a destination implementation for the requested provider.""" provider_lower: Final = provider.lower() @@ -39,7 +40,7 @@ class FocusDestinationFactory: def _resolve_config( *, provider: str, - overrides: dict[str, object], + overrides: Mapping[str, object], ) -> dict[str, Any]: if provider == "s3": resolved = { diff --git a/litellm/integrations/focus/export_engine.py b/litellm/integrations/focus/export_engine.py index 615aba72954..5c345630984 100644 --- a/litellm/integrations/focus/export_engine.py +++ b/litellm/integrations/focus/export_engine.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Callable -from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast +from collections.abc import Callable, Mapping +from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast import polars as pl @@ -24,7 +24,7 @@ class FocusExportEngine: provider: str, export_format: str, prefix: str, - destination_config: dict[str, Any] | None = None, + destination_config: Mapping[str, object] | None = None, ) -> None: self.provider = provider self.export_format = export_format @@ -77,7 +77,7 @@ class FocusExportEngine: return FocusParquetSerializer() raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.") - async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]: + async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]: data: Final = await self.database.get_usage_data(limit=limit) normalized: Final = self.transformer.transform(data) diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index e961b53a240..71b61296cfc 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -6,6 +6,8 @@ import os from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, ClassVar, Final, cast +from typing_extensions import TypedDict, Unpack + import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data" DEFAULT_DRY_RUN_LIMIT: Final = 500 +class _CustomLoggerOptions(TypedDict, total=False, extra_items=object): + pass + + class FocusLogger(CustomLogger): """Coordinates Focus export jobs across transformer/serializer/destination layers.""" @@ -40,7 +46,7 @@ class FocusLogger(CustomLogger): interval_seconds: int | None = None, prefix: str | None = None, destination_config: dict[str, Any] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomLoggerOptions], ) -> None: super().__init__(**kwargs) self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower() diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py index 9dfd75e5559..3f8cf93e11c 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py @@ -9,6 +9,7 @@ Usage: """ import asyncio +import os from collections.abc import AsyncIterable, Iterable from typing import Final @@ -40,7 +41,7 @@ _mocks_initialized = False # Default mock latency in seconds (simulates network round-trip) # Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE -_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 +_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None): diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index c799245c215..cbf6898d978 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -12,6 +12,8 @@ import os import traceback from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: @@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) +_DECODED_JSON: Final = TypeAdapter(object) + class GcsPubSubLogger(CustomBatchLogger): def __init__( @@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger): finally: self.log_queue.clear() - async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None: + async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object: """ Publish message to Google Cloud Pub/Sub using REST API @@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger): raise Exception(f"Failed to publish message: {response.text}") verbose_logger.debug("Pub/Sub response: %s", response.text) - return response.json() + return _DECODED_JSON.validate_python(response.json()) except Exception as e: verbose_logger.error("Pub/Sub publish error: %s", str(e)) diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 5d37894bb8f..a69c334a1fb 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str return (None if echoes_trace_id else explicit) or trace.session_id or None -def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: +def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None: """Seconds from the upstream request being issued (``api_call_start_time``) to the first streamed chunk (``completion_start_time``); ``None`` for non-streaming calls, where ``completion_start_time`` is backfilled with the @@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None - api_call_start: Final = to_seconds(kwargs.get("api_call_start_time")) - completion_start: Final = to_seconds(kwargs.get("completion_start_time")) + api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time")) + completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time")) if api_call_start is None or completion_start is None: return None return completion_start - api_call_start diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 7ec3711ee65..72d51471ba2 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False): response_cost: ReadOnly[float | None] completion_start_time: ReadOnly[datetime | float | str | None] api_call_start_time: ReadOnly[datetime | float | str | None] + end_time: ReadOnly[datetime | float | str | None] def resolve_error_type(kwargs: Mapping[str, Any]) -> str: @@ -414,15 +415,12 @@ class GenAIMetricRecorder: if generation_time > 0: self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs) - def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None: + def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None: api_call_start_time: Final = kwargs.get("api_call_start_time") if api_call_start_time is None: return - _end_time = kwargs.get("end_time") or end_time - if _end_time is None: - _end_time = datetime.now() api_call_start: Final = to_seconds(api_call_start_time) - end_ts: Final = to_seconds(_end_time) + end_ts: Final = to_seconds(kwargs.get("end_time") or end_time) if api_call_start is None or end_ts is None: return duration: Final = end_ts - api_call_start diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index b7c6f805233..544e9e81e46 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -631,11 +631,14 @@ def preserve_upstream_non_openai_attributes( for key in candidate_keys ) if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): - for key, value in candidate_values: - setattr(model_response, key, value) + _assign_attributes(model_response, candidate_values) return - for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items(): - setattr(model_response, key, value) + _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + + +def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: + for key, value in values: + setattr(target, key, value) def safe_deep_copy(data): @@ -761,6 +764,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: Filtered data structure with Exception and callable objects removed, or None if the entire input was an Exception or callable """ + return _filter_exceptions_from_params(data, max_depth) + + +def _filter_exceptions_from_params(data: object, max_depth: int) -> object: if max_depth <= 0: return data @@ -782,7 +789,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)): continue try: - filtered = filter_exceptions_from_params(v, max_depth - 1) + filtered = _filter_exceptions_from_params(v, max_depth - 1) if filtered is not None: result[k] = filtered except Exception: @@ -796,7 +803,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)): continue try: - filtered = filter_exceptions_from_params(item, max_depth - 1) + filtered = _filter_exceptions_from_params(item, max_depth - 1) if filtered is not None: result_list.append(filtered) except Exception: diff --git a/litellm/litellm_core_utils/cost_discount.py b/litellm/litellm_core_utils/cost_discount.py index a64ad0b20f7..bfcee7928fb 100644 --- a/litellm/litellm_core_utils/cost_discount.py +++ b/litellm/litellm_core_utils/cost_discount.py @@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey: def _literal_length(pattern: str) -> int: - return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?") + return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?") def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]: diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index c3b6a008411..970dd512a40 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -48,5 +48,5 @@ for attempt in range(_max_retries): # Last attempt, re-raise the exception raise # Exponential backoff with jitter to reduce collision probability - delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1) + delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1) time.sleep(delay) diff --git a/litellm/litellm_core_utils/dot_notation_indexing.py b/litellm/litellm_core_utils/dot_notation_indexing.py index 1dac67ecbf6..09720fa413c 100644 --- a/litellm/litellm_core_utils/dot_notation_indexing.py +++ b/litellm/litellm_core_utils/dot_notation_indexing.py @@ -181,7 +181,7 @@ def _delete_nested_value_custom( def delete_nested_value( - data: dict[str, Any], + data: dict[str, object], path: str, depth: int = 0, max_depth: int = 20, diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 7b3835dc1ac..a162ad94510 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +_DECODED_JSON: Final = TypeAdapter(object) _BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -313,7 +314,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) - def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome: try: - parsed: Final[object] = json.loads(content) + parsed: Final = _DECODED_JSON.validate_python(json.loads(content)) except ValueError as e: return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): @@ -382,7 +383,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp ) if response.is_error: return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") - return _classify_fetch_content(response.content, url, response.headers.get("etag")) + etag_values: Final = response.headers.get_list("etag") + return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None) def _next_retry_wait( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e2eed3cf6d5..1904885cd59 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -551,6 +551,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_REDACTED_RESULT: Final = TypeAdapter(object) def _provider_response_id(source: object) -> str | None: @@ -1714,7 +1715,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) - original_response = redact_message_input_output_from_logging( + redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), result=original_response, ) @@ -1941,10 +1942,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details.pop("response_cost_failure_debug_information", None) - if cache_hit is None: - cache_hit = self.model_call_details.get("cache_hit", False) + resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit - if cache_hit is True: + if resolved_cache_hit is True: return 0.0 if is_unbilled_non_inference_call( @@ -1996,7 +1996,7 @@ class Logging(LiteLLMLoggingBaseClass): model_value: Final = litellm_model_name or self.model cost_cache_hit_value: Final = ( - self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit + self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit ) cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided bool | None, cost_cache_hit_value @@ -2375,8 +2375,10 @@ class Logging(LiteLLMLoggingBaseClass): def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" - usage: Final = self.model_call_details.pop("combined_usage_object", None) - response_cost: Final = self.model_call_details.pop("response_cost", None) + usage: Final = self._model_call_details_view.get("combined_usage_object") + response_cost: Final = self._model_call_details_view.get("response_cost") + self.model_call_details.pop("combined_usage_object", None) + self.model_call_details.pop("response_cost", None) if isinstance(usage, Usage): self.model_call_details["failed_attempts_usage"] = ( *self.model_call_details.get("failed_attempts_usage", ()), @@ -3170,9 +3172,11 @@ class Logging(LiteLLMLoggingBaseClass): ) ## REDACT MESSAGES ## - result = redact_message_input_output_from_logging( - model_call_details=call_details, - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=call_details, + result=result, + ) ) ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) @@ -3438,7 +3442,7 @@ class Logging(LiteLLMLoggingBaseClass): print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response") s3Logger.log_event( kwargs=call_details, - response_obj=call_details["complete_streaming_response"], + response_obj=call_details_view["complete_streaming_response"], start_time=start_time, end_time=end_time, print_verbose=print_verbose, @@ -3754,9 +3758,11 @@ class Logging(LiteLLMLoggingBaseClass): self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks ) - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) ## LOGGING HOOK ## @@ -4094,9 +4100,11 @@ class Logging(LiteLLMLoggingBaseClass): result: object = None # result sent to all loggers, init this to None incase it's not created - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) self.mark_logging_complete(event_type="sync_failure") for callback in callbacks: diff --git a/litellm/litellm_core_utils/llm_judge.py b/litellm/litellm_core_utils/llm_judge.py index 0c2d7767455..307e000119b 100644 --- a/litellm/litellm_core_utils/llm_judge.py +++ b/litellm/litellm_core_utils/llm_judge.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def default_router_provider() -> Router | None: @@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload parsed: object try: - parsed = json.loads(text) + parsed = _DECODED_JSON.validate_python(json.loads(text)) except json.JSONDecodeError: start: Final = text.find("{") end: Final = text.rfind("}") if start == -1 or end <= start: raise - parsed = json.loads(text[start : end + 1]) + parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1])) if not isinstance(parsed, dict): raise ValueError("judge response is not a JSON object") return {str(k): v for k, v in parsed.items()} diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 9f521ffc6cc..eb5bebd6f3f 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -1,4 +1,4 @@ -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final import litellm @@ -17,6 +17,10 @@ else: _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {} +def _instance_attributes(target: object) -> Mapping[str, object]: + return vars(target) + + class LoggingCallbackManager: """ A centralized class that allows easy add / remove callbacks for litellm. @@ -329,7 +333,7 @@ class LoggingCallbackManager: key_parts: Final = [type(custom_logger).__name__] # Add only fundamental type instance variables to the key - for attr_name, attr_value in vars(custom_logger).items(): + for attr_name, attr_value in _instance_attributes(custom_logger).items(): if not attr_name.startswith("_"): # Skip private attributes if isinstance(attr_value, (str, bool, int)): key_parts.append(f"{attr_name}={attr_value}") diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b704dfa2675..ea336f899f7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1527,7 +1527,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int: if properties is None: return 0 optional: Final = len(_optional_property_names(node)) - return 2**optional * (len(properties) - optional + 1) + return (1 << optional) * (len(properties) - optional + 1) def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 481ec6a5076..61c1fc5f1d4 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3086,10 +3086,14 @@ def anthropic_messages_pt( def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) if strip: - ext_list = [e.strip() for e in ext_list] + ext_list = list(_stripped(ext_list)) return ext_list +def _stripped(values: Iterable[str]) -> Iterator[str]: + return (value.strip() for value in values) + + def contains_tag(tag: str, string: str) -> bool: return bool(re.search(f"<{tag}>(.+?)", string, re.DOTALL)) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 52758e617d8..8c402fe9b44 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import ItemsView, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field from typing import Any, Final @@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( ) +def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]: + return data.items() + + class SensitiveDataMasker: def __init__( self, @@ -127,7 +131,7 @@ class SensitiveDataMasker: return data masked_data: Final[dict[str, object]] = {} - for k, v in data.items(): + for k, v in _object_items(data): try: key_is_sensitive = self.is_sensitive_key(k, excluded_keys) if isinstance(v, Mapping): diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6ec426d6da5..470892fe2fc 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -108,8 +108,7 @@ def _prepend_upstream_choice_extras( choice.__pydantic_fields_set__.update(upstream_extras) -def _holds_only_scalars(model: BaseModel) -> bool: - fields: Final[Iterator[tuple[str, object]]] = iter(model) +def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool: return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) @@ -940,8 +939,8 @@ class CustomStreamWrapper: or ("function_call" in completion_obj and completion_obj["function_call"] is not None) or ( "tool_calls" in model_response.choices[0].delta - and model_response.choices[0].delta["tool_calls"] is not None - and len(model_response.choices[0].delta["tool_calls"]) > 0 + and model_response.choices[0].delta.tool_calls is not None + and len(model_response.choices[0].delta.tool_calls) > 0 ) or ( "function_call" in model_response.choices[0].delta diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index 3f3716e0de6..a9d9f930b9e 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -132,7 +132,7 @@ class AnthropicBatchesHandler: max_retries: int | None, logging_obj: LiteLLMLoggingObj | None = None, litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment - ) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: + ) -> LiteLLMBatch | Coroutine[object, object, LiteLLMBatch]: """ Retrieve a batch from Anthropic. diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index aa2bee1f31d..d492edf9d4b 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -1629,14 +1629,13 @@ class AnthropicMessagesHandler(BaseTranslation): if not line.startswith("data:"): return line try: - data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads( - line[len("data:") :].strip() - ) + data: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip())) except json.JSONDecodeError: return line - if not isinstance(data, dict): + event: Final = as_str_mapping(data) + if not isinstance(event, dict): return line - rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event) + rewritten: Final = _rewritten_event(event, rewrite_event) return line if rewritten is data else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 935ce77ef87..d4430ca5ab9 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from functools import reduce from itertools import chain from types import MappingProxyType -from typing import Annotated, Any, Final, Literal, TypeVar, cast +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -32,6 +32,9 @@ from ..constants import ( from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +if TYPE_CHECKING: + from litellm.types.llms.openai import ChatCompletionToolParam + T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 @@ -216,7 +219,7 @@ def _trigger_met( current_tokens: Final = litellm.token_counter( model=model, messages=messages, - tools=cast(Any, tools), + tools=cast("list[ChatCompletionToolParam] | None", tools), ) verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) verbose_logger.debug("context_management polyfill: threshold: %s", threshold) @@ -366,9 +369,13 @@ def apply_clear_tool_uses_20250919( tokens_before: Final = ( counted_tokens if counted_tokens is not None - else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + else litellm.token_counter( + model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools) + ) + ) + tokens_after: Final = litellm.token_counter( + model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools) ) - tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/pass_through/context_management/placeholders.py b/litellm/llms/anthropic/pass_through/context_management/placeholders.py index b3d3529f105..8a0eb6d48e1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/placeholders.py +++ b/litellm/llms/anthropic/pass_through/context_management/placeholders.py @@ -1,12 +1,10 @@ """Placeholder content for cleared ``tool_result`` blocks (string or block list).""" -from typing import Any - from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER def build_cleared_tool_result_content( - original_content: Any, + original_content: object, ) -> str | list[dict]: """Return a string or single text block list, matching ``original_content`` shape.""" if isinstance(original_content, list): diff --git a/litellm/llms/anthropic/pass_through/context_management/result.py b/litellm/llms/anthropic/pass_through/context_management/result.py index 5b88fe30871..4713b8d79c7 100644 --- a/litellm/llms/anthropic/pass_through/context_management/result.py +++ b/litellm/llms/anthropic/pass_through/context_management/result.py @@ -6,7 +6,7 @@ attach ``iterations`` to ``usage``. """ from dataclasses import dataclass, field -from typing import Any, Final +from typing import Final from litellm.types.llms.anthropic import ( AppliedEdit, @@ -19,8 +19,8 @@ from .constants import COMPACT_EDIT_TYPE @dataclass class PolyfillResult: - messages: list[dict[str, Any]] - system: str | list[dict[str, Any]] | None + messages: list[dict[str, object]] + system: str | list[dict[str, object]] | None applied_edits: list[AppliedEdit] = field(default_factory=list) compaction_block: CompactionBlock | None = None iterations_usage: list[UsageIteration] | None = None diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py index 8d1b358c8f1..2b6161d98f6 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py @@ -9,8 +9,8 @@ the LLM doesn't make a tool call, and we need to return a stream to the user. """ import json -from collections.abc import Mapping -from typing import Any, Final, cast +from collections.abc import Mapping, Sequence +from typing import Final, cast from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -136,8 +136,7 @@ class FakeAnthropicMessagesStreamIterator: """Convert the non-streaming response to streaming chunks""" chunks: Final = [] - # Cast response to dict for easier access - response_dict: Final = cast(dict[str, Any], self.response) + response_dict: Final[Mapping[str, object]] = self.response # 1. message_start event usage: Final = self.response.get("usage") @@ -160,9 +159,8 @@ class FakeAnthropicMessagesStreamIterator: chunks.append(f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode()) # 2-4. For each content block, send start/delta/stop events - content_blocks: Final = response_dict.get("content", []) - for index, block in enumerate(content_blocks): - block_dict = cast(dict[str, Any], block) + content_blocks: Final = cast(Sequence[Mapping[str, object]], response_dict.get("content", [])) + for index, block_dict in enumerate(content_blocks): chunks.extend(self._create_content_block_chunks(block_dict, index)) # 5. message_delta event (with final usage and stop_reason) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index 0fafd4abd62..c070b462663 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -242,7 +242,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _summary_part_text(part: object) -> str: if isinstance(part, Mapping): - mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json + mapping: Final = cast(Mapping[str, object], part) # cast-ok: summary parts are untyped provider json return str(mapping.get("text") or "") return str(getattr(part, "text", None) or "") @@ -471,7 +471,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: """Convert Anthropic tool definitions to Responses API function tools.""" result: Final[list[dict[str, object]]] = [] for tool in tools: - tool_dict = cast(dict[str, Any], tool) + tool_dict = cast(Mapping[str, object], tool) tool_type = tool_dict.get("type", "") tool_name = tool_dict.get("name", "") # web_search tool @@ -674,7 +674,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config = anthropic_request.get("output_config") reasoning: Final = self.translate_thinking_to_reasoning( thinking, - output_config=cast(dict[str, Any] | None, output_config), + output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, drop_params=drop_params, ) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 99ebd2875a7..df8a0614665 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from openai import AsyncAzureOpenAI, AzureOpenAI from pydantic import BaseModel @@ -38,7 +38,7 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: + ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} if atranscription is True: diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 67b25bf7d12..617f998a4ca 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -13,7 +13,7 @@ from openai import ( BadRequestError, OpenAI, ) -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES @@ -59,6 +59,8 @@ from .image_generation import ( ) from .image_generation.http_utils import azure_deployment_image_generation_json_body +_DECODED_JSON: Final = TypeAdapter(object) + class AzureOpenAIAssistantsAPIConfig: """ @@ -961,7 +963,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, @@ -1078,7 +1080,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 74e65572809..86ae8f5c224 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -94,7 +94,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_stripped_model_name(self, model: str) -> str: return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: """ Handle reasoning items to filter out the status field. Issue: https://github.com/BerriAI/litellm/issues/13484 diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 0fa54cccbc9..f1d062b1645 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -396,7 +396,7 @@ class AzureAIStudioConfig(OpenAIConfig): return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: - fields_to_drop: Final = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) + fields_to_drop: Final[Collection[str]] = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) tools: Final = request_data.get("tools") if fields_to_drop and isinstance(tools, list): for tool in tools: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 51c8e4b8bef..a97a8be6253 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -428,7 +428,7 @@ class BaseConfig(ABC): streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse"], sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> object: pass def get_model_response_iterator_for_request( @@ -513,7 +513,7 @@ class BaseConfig(ABC): def apply_assembled_streaming_response_metadata( self, response: "ModelResponse", - chunks: list[Any], + chunks: list[object], ) -> None: """Hook for providers to merge chunk metadata into assembled streaming responses.""" return diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index ee2f6eb9f1a..1805515ad77 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -121,7 +121,7 @@ class BaseTranslation(ABC): @staticmethod def merge_user_api_key_metadata_into_request( - request_data: dict[str, Any], # mutable-ok: proxy hooks share and mutate the request payload dict in place + request_data: dict[str, object], # mutable-ok: proxy hooks share and mutate the request payload dict in place user_api_key_dict: Optional["UserAPIKeyAuth"], ) -> None: """ diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index bb4cf42fa1e..9ffda853e8b 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -141,7 +141,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): self, llm_router: Router, model: str, - request_data: dict[str, Any], + request_data: dict[str, object], litellm_parent_otel_span: Span, ) -> ResourceObjectType: """ @@ -399,7 +399,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): async def create_resource_for_each_model( self, llm_router: Router, - request_data: dict[str, Any], + request_data: dict[str, object], target_model_names_list: list[str], litellm_parent_otel_span: Span, ) -> list[ResourceObjectType]: @@ -550,7 +550,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): limit: int | None = None, after: str | None = None, additional_filters: Mapping[str, object] | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ List resources created by a user. diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 28f6348e4d9..138700b3212 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -152,7 +152,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: pass @@ -164,7 +164,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Optional async version of transform_search_vector_store_request. diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 48b3795e274..0a645b90c29 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -50,6 +51,8 @@ else: HTTPHandler = Any AsyncHTTPHandler = Any +_DECODED_JSON: Final = TypeAdapter(object) + class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): def __init__(self, **kwargs): @@ -313,7 +316,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return None try: - data: Final[object] = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: @@ -481,7 +484,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") - response_json: Final[object] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: @@ -565,7 +568,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -705,7 +708,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = response.read() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, @@ -787,7 +790,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -927,7 +930,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = await response.aread() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index e48196946cd..d61541aab8a 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -21,6 +21,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.anthropic.chat.handler import ModelResponseIterator LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -134,7 +135,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "ModelResponseIterator": return self.get_model_response_iterator_for_request( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, optional_params={} ) @@ -145,7 +146,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): sync_stream: bool, json_mode: bool | None, optional_params: Mapping[str, object], - ) -> object: + ) -> "ModelResponseIterator": from litellm.llms.anthropic.chat.handler import ModelResponseIterator return ModelResponseIterator( diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 3d23b69f846..37f95ff7b9b 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final import httpx @@ -139,7 +139,7 @@ class BedrockFilesHandler(BaseAWSLLM): optional_params: dict, timeout: float | httpx.Timeout, max_retries: int | None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from S3 bucket for Bedrock files. Supports both sync and async operations. diff --git a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py index 145e6a0781f..5731541da56 100644 --- a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py @@ -3,6 +3,7 @@ Transformation logic for Amazon Titan Image Generation. """ import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -35,16 +36,17 @@ class AmazonTitanImageGenerationConfig: width: int | None = None, height: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index a90b4829634..1b01b0ec6d0 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -544,7 +544,7 @@ class _StreamFrame(TypedDict): def _unpack_uint32(buffer: bytes) -> int: - return struct.unpack("!I", buffer)[0] + return int.from_bytes(buffer, "big") def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]: @@ -1255,7 +1255,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): headers_bytes = frame_raw[12 : 12 + orig_hdrs_len] try: - payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + payload_dict = _BODY_BY_REFERENCE.validate_python( + json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + ) for local_idx, (_, container, key) in enumerate( _collect_stream_delta_text_holders(payload_dict.get("delta")) ): diff --git a/litellm/llms/black_forest_labs/image_edit/transformation.py b/litellm/llms/black_forest_labs/image_edit/transformation.py index 66eacc97fba..44f5243e927 100644 --- a/litellm/llms/black_forest_labs/image_edit/transformation.py +++ b/litellm/llms/black_forest_labs/image_edit/transformation.py @@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict): _BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult) +_DECODED_JSON: Final = TypeAdapter(object) _BFL_REQUEST_PARAMS: Final = ( @@ -317,7 +318,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - raw_response_data: Final[object] = raw_response.json() + raw_response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index 7c985b7dd5e..a1aac7f315b 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -72,7 +72,7 @@ class ChatGPTConfig(OpenAIConfig): default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id) return {**default_headers, **validated_headers} - def post_stream_processing(self, stream: Any) -> Any: + def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer: return ChatGPTToolCallNormalizer(stream) def map_openai_params( diff --git a/litellm/llms/cometapi/chat/transformation.py b/litellm/llms/cometapi/chat/transformation.py index 26050700efc..c2afad16d3a 100644 --- a/litellm/llms/cometapi/chat/transformation.py +++ b/litellm/llms/cometapi/chat/transformation.py @@ -6,7 +6,7 @@ Documentation: [CometAPI Documentation Link] """ from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from typing import Final import httpx @@ -138,7 +138,7 @@ class CometAPIConfig(OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "CometAPIChatCompletionStreamingHandler": """ Get model response iterator for streaming responses """ diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index a02db2338d8..8943481eb3a 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -2,10 +2,11 @@ CompactifAI chat completion transformation """ -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Iterable, Mapping, Sequence, Sized +from typing import TYPE_CHECKING, Any, Final, Protocol import httpx +from pydantic import InstanceOf, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -24,6 +25,21 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]]) +_JSON_VALUES: Final = TypeAdapter(Iterable[object]) +_JSON_OBJECT: Final = TypeAdapter(InstanceOf[Mapping[str, object]]) +_MESSAGE: Final = TypeAdapter(InstanceOf[dict[str, object]]) +_SIZED: Final = TypeAdapter(InstanceOf[Sized]) +_TOOL_CALLS: Final = TypeAdapter(InstanceOf[Sequence[object]]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + class CompactifAIResponseFields(TypedDict, total=False): """The chat completion fields of a CompactifAI response body.""" @@ -82,22 +98,23 @@ class CompactifAIChatConfig(OpenAIGPTConfig): ) ## RESPONSE OBJECT - response_json: Final = raw_response.json() + response_json: Final = _RESPONSE_BODY.validate_python(_json_body(raw_response)) # Handle JSON mode if needed if json_mode: - for choice in response_json["choices"]: - message = choice.get("message") - if message and message.get("tool_calls"): + for choice in _JSON_VALUES.validate_python(response_json["choices"]): + message = _JSON_OBJECT.validate_python(choice).get("message") + message_fields = _MESSAGE.validate_python(message) if message else None + if message_fields is not None and message_fields.get("tool_calls"): # Convert tool calls to content for JSON mode - tool_calls = message.get("tool_calls", []) - if len(tool_calls) == 1: - message["content"] = tool_calls[0]["function"].get("arguments", "") - message["tool_calls"] = None + tool_calls = message_fields.get("tool_calls", []) + if len(_SIZED.validate_python(tool_calls)) == 1: + first_call = _JSON_OBJECT.validate_python(_TOOL_CALLS.validate_python(tool_calls)[0]) + function = _JSON_OBJECT.validate_python(first_call["function"]) + message_fields["content"] = function.get("arguments", "") + message_fields["tool_calls"] = None - response_fields: Final[CompactifAIResponseFields] = response_json - - returned_response: Final = ModelResponse(**response_fields) + returned_response: Final = ModelResponse(**response_json) # Set model name with provider prefix returned_response.model = f"compactifai/{model}" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 48858dfa441..c80e7dcd4a7 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -407,6 +407,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _run_responses_deployment_hook( + *, + model: str, + request_input: str | ResponseInputParam, + custom_llm_provider: str, + optional_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, +) -> Mapping[str, object]: + return run_async_function( + async_pre_call_deployment_hook, + { + **dict(litellm_params), + **optional_params, + "model": model, + "input": request_input, + "custom_llm_provider": custom_llm_provider, + }, + CallTypes.responses.value, + ) + + def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict: """A pre-signed request carries its auth inside its own ``headers`` key, which logging treats as request body (only the top-level headers channel gets masked), @@ -2637,16 +2658,12 @@ class BaseLLMHTTPHandler: litellm_params, ) - modified_kwargs: Final = run_async_function( - async_pre_call_deployment_hook, - { - **dict(litellm_params), - **response_api_optional_request_params, - "model": model, - "input": input, - "custom_llm_provider": custom_llm_provider, - }, - CallTypes.responses.value, + modified_kwargs: Final = _run_responses_deployment_hook( + model=model, + request_input=input, + custom_llm_provider=custom_llm_provider, + optional_params=response_api_optional_request_params, + litellm_params=litellm_params, ) if modified_kwargs is None: return ( diff --git a/litellm/llms/custom_httpx/mock_transport.py b/litellm/llms/custom_httpx/mock_transport.py index 0322cceec0f..8b99b18a97f 100644 --- a/litellm/llms/custom_httpx/mock_transport.py +++ b/litellm/llms/custom_httpx/mock_transport.py @@ -9,9 +9,11 @@ so the full proxy -> router -> OpenAI SDK -> httpx path is exercised. import json import time import uuid +from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter # --------------------------------------------------------------------------- # Pre-built response templates @@ -22,7 +24,7 @@ def _mock_id() -> str: return f"chatcmpl-mock-{uuid.uuid4().hex[:8]}" -def _chat_completion_json(model: str) -> dict: +def _chat_completion_json(model: object) -> dict: """Return a minimal valid ChatCompletion object.""" return { "id": _mock_id(), @@ -55,6 +57,9 @@ _JSON_HEADERS: Final = { "content-type": "application/json", } +_DECODED_JSON: Final = TypeAdapter(object) +_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) + class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @@ -64,13 +69,13 @@ class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @staticmethod - def _parse_request(request: httpx.Request) -> tuple[str, bool]: + def _parse_request(request: httpx.Request) -> tuple[object, bool]: """Extract model from the request body.""" try: - body: Final = json.loads(request.content) + body: Final = _DECODED_JSON.validate_python(json.loads(request.content)) except (json.JSONDecodeError, ValueError): return ("mock-model", False) - model: Final = body.get("model", "mock-model") + model: Final = _REQUEST_BODY.validate_python(body).get("model", "mock-model") return (model, False) async def handle_async_request(self, request: httpx.Request) -> httpx.Response: diff --git a/litellm/llms/databricks/embed/transformation.py b/litellm/llms/databricks/embed/transformation.py index ddaf2bbb621..0feb2aea5a0 100644 --- a/litellm/llms/databricks/embed/transformation.py +++ b/litellm/llms/databricks/embed/transformation.py @@ -3,6 +3,7 @@ Translates from OpenAI's `/v1/embeddings` to Databricks' `/embeddings` """ import types +from collections.abc import Mapping from typing import Final @@ -16,16 +17,17 @@ class DatabricksEmbeddingConfig: ) def __init__(self, instruction: str | None = None) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/deepagents/harness/sandbox_backend.py b/litellm/llms/deepagents/harness/sandbox_backend.py index d39690ccdd1..8ad90faa2b4 100644 --- a/litellm/llms/deepagents/harness/sandbox_backend.py +++ b/litellm/llms/deepagents/harness/sandbox_backend.py @@ -22,7 +22,7 @@ import shlex import uuid from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import Any, Final, TypeVar +from typing import Final, TypeVar from deepagents.backends.protocol import ( DeleteResult, @@ -171,7 +171,7 @@ class SandboxBackend(SandboxBackendProtocol): # pyright: ignore[reportUntypedBa # -- sync bridge (deepagents only calls these outside the event loop) --- - def _sync(self, coro: Coroutine[Any, Any, T]) -> T: + def _sync(self, coro: Coroutine[object, object, T]) -> T: try: running = asyncio.get_running_loop() except RuntimeError: diff --git a/litellm/llms/deepagents/harness/transformation.py b/litellm/llms/deepagents/harness/transformation.py index cb80e3e12f7..54a6a5573ac 100644 --- a/litellm/llms/deepagents/harness/transformation.py +++ b/litellm/llms/deepagents/harness/transformation.py @@ -11,11 +11,13 @@ from __future__ import annotations import itertools import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol +from pydantic import TypeAdapter + from litellm.harness.options import DeepAgentsOptions from litellm.harness.types import ( Capabilities, @@ -41,6 +43,7 @@ DEEPAGENTS_DEFAULT_RECURSION_LIMIT: Final = 1000 _MODEL_NODE: Final = "model" # Only these graph nodes produce new messages; middleware hooks may re-emit history. _EVENT_NODES: Final = frozenset({"model", "tools"}) +_TOOL_CALLS: Final = TypeAdapter(Iterable[Mapping[str, object]]) NORMALIZED_TO_NATIVE: Final[Mapping[str, str]] = MappingProxyType( { @@ -208,7 +211,7 @@ def update_events( def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event, ...]: kind = getattr(message, "type", None) if kind == "ai": - calls = getattr(message, "tool_calls", None) or () + calls: Final = _TOOL_CALLS.validate_python(getattr(message, "tool_calls", None) or ()) return tuple(tool_call_event(call) for call in calls if call.get("name") not in skip_tools) if kind == "tool" and getattr(message, "name", None) not in skip_tools: status: Final[object] = getattr(message, "status", None) diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index a3812dac8ef..4d8dd4d0b55 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -2,12 +2,12 @@ Translates from OpenAI's `/v1/audio/transcriptions` to Deepgram's `/v1/listen` """ -from collections.abc import Iterable, Mapping -from typing import Final +from collections.abc import Container, Iterable, Mapping, Sequence, Sized +from typing import Final, Protocol from urllib.parse import urlencode from httpx import Headers, Response -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, InstanceOf, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -26,6 +26,19 @@ from ..common_utils import DeepgramException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_ITEMS: Final = TypeAdapter(InstanceOf[Sequence[object]], config=ConfigDict(hide_input_in_errors=True)) +_SIZED: Final = TypeAdapter(InstanceOf[Sized], config=ConfigDict(hide_input_in_errors=True)) +_CONTAINER: Final = TypeAdapter(InstanceOf[Container[object]], config=ConfigDict(hide_input_in_errors=True)) +_WORD_LIST: Final = TypeAdapter(InstanceOf[list[object]], config=ConfigDict(hide_input_in_errors=True)) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -81,16 +94,21 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from Deepgram to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + response_json: Final = _RESPONSE_BODY.validate_python(_json_body(raw_response)) # Get the first alternative from the first channel - first_channel: Final = response_json["results"]["channels"][0] - first_alternative: Final = first_channel["alternatives"][0] + results: Final = _JSON_OBJECT.validate_python(response_json["results"]) + first_channel: Final = _JSON_OBJECT.validate_python(_ITEMS.validate_python(results["channels"])[0]) + first_alternative: Final = _JSON_OBJECT.validate_python( + _ITEMS.validate_python(first_channel["alternatives"])[0] + ) # Detect if diarization is active by checking if words have 'speaker' field has_diarization = False - if "words" in first_alternative and len(first_alternative["words"]) > 0: - has_diarization = "speaker" in first_alternative["words"][0] + if "words" in first_alternative and len(_SIZED.validate_python(first_alternative["words"])) > 0: + has_diarization = "speaker" in _CONTAINER.validate_python( + _ITEMS.validate_python(first_alternative["words"])[0] + ) # Extract the transcript based on diarization mode if not has_diarization: @@ -98,10 +116,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): text = first_alternative["transcript"] elif "paragraphs" in first_alternative: # Diarization with paragraphs: use the pre-formatted diarized transcript - text = first_alternative["paragraphs"]["transcript"] + text = _JSON_OBJECT.validate_python(first_alternative["paragraphs"])["transcript"] else: # Diarization without paragraphs: reconstruct from words - text = self._reconstruct_diarized_transcript(first_alternative["words"]) + text = self._reconstruct_diarized_transcript(_WORD_LIST.validate_python(first_alternative["words"])) # Create TranscriptionResponse object response: Final = TranscriptionResponse(text=text) @@ -110,10 +128,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response["task"] = "transcribe" # Use detected_language if available, otherwise default to "en" - detected_language: Final = _JSON_OBJECT.validate_python(first_channel).get("detected_language") + detected_language: Final = first_channel.get("detected_language") response["language"] = detected_language if detected_language else "en" - response["duration"] = response_json["metadata"]["duration"] + response["duration"] = _JSON_OBJECT.validate_python(response_json["metadata"])["duration"] # Transform words to match OpenAI format if "words" in first_alternative: diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 6a6b57131ad..f310923c2ec 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -2,8 +2,10 @@ DeepSeek Anthropic-compatible messages transformation config. """ -from collections.abc import Sequence -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter import litellm from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -12,6 +14,24 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams +_TOOL_LIST: Final = TypeAdapter(list[object]) +_TOOL_FIELDS: Final = TypeAdapter(dict[object, object]) + + +def _custom_tool_without_type(tool: object) -> Mapping[object, object] | None: + if not isinstance(tool, dict): + return None + fields: Final = _TOOL_FIELDS.validate_python(tool) + if fields.get("type") == "custom": + fields.pop("type", None) + return fields + return None + + +def _sanitized_tool(tool: object) -> object: + stripped: Final = _custom_tool_without_type(tool) + return tool if stripped is None else stripped + class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): """ @@ -95,19 +115,11 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): return f"{base_url}/v1/messages" @staticmethod - def _sanitize_tools_for_deepseek(tools: Any) -> object: + def _sanitize_tools_for_deepseek(tools: object) -> object: if not isinstance(tools, list): return tools - sanitized_tools: Final = [] - for tool in tools: - if isinstance(tool, dict) and tool.get("type") == "custom": - sanitized_tool = dict(tool) - sanitized_tool.pop("type", None) - sanitized_tools.append(sanitized_tool) - else: - sanitized_tools.append(tool) - return sanitized_tools + return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)] def transform_anthropic_messages_request( self, diff --git a/litellm/llms/deprecated_providers/palm.py b/litellm/llms/deprecated_providers/palm.py index 785cffa48ea..b4a32630c42 100644 --- a/litellm/llms/deprecated_providers/palm.py +++ b/litellm/llms/deprecated_providers/palm.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final @@ -41,16 +42,17 @@ class PalmConfig: top_p: float | None = None, max_output_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/edenai/embedding/transformation.py b/litellm/llms/edenai/embedding/transformation.py index c79a6839434..a8e46fd2293 100644 --- a/litellm/llms/edenai/embedding/transformation.py +++ b/litellm/llms/edenai/embedding/transformation.py @@ -5,7 +5,7 @@ per-request cost at the top level of the body. Docs: https://www.edenai.co/docs/v3/llms/embeddings """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol import httpx @@ -24,6 +24,14 @@ if TYPE_CHECKING: _SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user") +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIEmbeddingConfig(BaseEmbeddingConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract return list(_SUPPORTED_PARAMS) @@ -80,12 +88,15 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract ) -> EmbeddingResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) + cost: Final = reported_cost(body) response: Final[EmbeddingResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="embedding" + response_object=body if isinstance(body, dict) else None, + model_response_object=model_response, + response_type="embedding", ) - set_response_cost_in_hidden_params(response, reported_cost(body)) + set_response_cost_in_hidden_params(response, cost) return response def get_error_class( diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index e2a0b8684f6..83cb1b19aa0 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -5,7 +5,7 @@ for every image model in the catalog with the real per-request cost at the top l Docs: https://www.edenai.co/docs/v3/llms/image-generation """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol import httpx @@ -36,6 +36,14 @@ _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( ) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIImageGenerationConfig(BaseImageGenerationConfig): def get_supported_openai_params( self, model: str @@ -97,12 +105,15 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) + cost: Final = reported_cost(body) response: Final[ImageResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="image_generation" + response_object=body if isinstance(body, dict) else None, + model_response_object=model_response, + response_type="image_generation", ) - set_response_cost_in_hidden_params(response, reported_cost(body)) + set_response_cost_in_hidden_params(response, cost) return response def get_error_class( diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 04d2717d64f..3870c3f8342 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/audio/transcriptions` to ElevenLabs's `/v1/speech- """ from collections.abc import Iterable, Mapping -from typing import Final +from typing import Final, Protocol from httpx import Headers, Response from pydantic import ConfigDict, TypeAdapter @@ -26,6 +26,15 @@ from ..common_utils import ElevenLabsException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -119,7 +128,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from ElevenLabs to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) response_object: Final = _JSON_OBJECT.validate_python(response_json) # Extract the main transcript text @@ -134,20 +143,18 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): # Map ElevenLabs words to OpenAI format if "words" in response_object: - response["words"] = [] - for word_data in _JSON_OBJECTS.validate_python(response_object["words"]): - # Only include actual words, skip spacing and audio events - if word_data.get("type") == "word": - response["words"].append( - { - "word": word_data.get("text", ""), - "start": word_data.get("start", 0), - "end": word_data.get("end", 0), - } - ) + response["words"] = [ + { + "word": word_data.get("text", ""), + "start": word_data.get("start", 0), + "end": word_data.get("end", 0), + } + for word_data in _JSON_OBJECTS.validate_python(response_object["words"]) + if word_data.get("type") == "word" + ] # Store full response in hidden params - response.hidden_params = response_json + response.hidden_params = _HIDDEN_PARAMS.validate_python(response_json) return response diff --git a/litellm/llms/gemini/agents/transformation.py b/litellm/llms/gemini/agents/transformation.py index cfdb55f2048..d11ddfe59b1 100644 --- a/litellm/llms/gemini/agents/transformation.py +++ b/litellm/llms/gemini/agents/transformation.py @@ -10,9 +10,10 @@ Proxies the Gemini v1beta Agents API: """ from collections.abc import Mapping -from typing import Any, Final +from typing import Any, Final, Protocol import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig @@ -28,6 +29,18 @@ from litellm.types.agents import ( # create-agent body verbatim. _GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment") +_AGENT_PAYLOAD: Final = TypeAdapter(dict[str, object]) +_AGENT_LISTING: Final = TypeAdapter(Mapping[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + # LiteLLM-internal keys that must never be forwarded to Gemini. _LITELLM_INTERNAL_KEYS: Final = frozenset( { @@ -155,18 +168,19 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data: dict[str, Any] = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: verbose_logger.warning( "GeminiAgentsConfig: non-JSON create response (status=%d).", raw_response.status_code, ) - data = {"id": name} + decoded = {"id": name} + data: Final = _AGENT_PAYLOAD.validate_python(decoded) # Gemini uses "id" as the identifier; normalise to both fields. data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig create response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # LIST # @@ -191,13 +205,16 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): ) -> AgentListResponse: self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = _AGENT_LISTING.validate_python(decoded) verbose_logger.debug("GeminiAgentsConfig list response: %s", data) - return AgentListResponse( - agents=data.get("agents", []), - next_page_token=data.get("nextPageToken"), + return AgentListResponse.model_validate( + { + "agents": data.get("agents", []), + "next_page_token": data.get("nextPageToken"), + } ) # ------------------------------------------------------------------ # @@ -221,13 +238,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """Same shape as create response — Gemini returns "id" as identifier.""" self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {"id": name} + decoded = {"id": name} + data: Final = _AGENT_PAYLOAD.validate_python(decoded) data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig get response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # DELETE # @@ -284,11 +302,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = _AGENT_LISTING.validate_python(decoded) verbose_logger.debug("GeminiAgentsConfig list_versions response for '%s': %s", name, data) - return AgentVersionsResponse( - agent_versions=data.get("agentVersions", []), - next_page_token=data.get("nextPageToken"), + return AgentVersionsResponse.model_validate( + { + "agent_versions": data.get("agentVersions", []), + "next_page_token": data.get("nextPageToken"), + } ) diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index c5ebd84e00f..9495c7cb661 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -2,11 +2,11 @@ import base64 import datetime import json import math -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from typing import Any, Final import httpx -from pydantic import JsonValue, TypeAdapter +from pydantic import InstanceOf, JsonValue, TypeAdapter import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -130,6 +130,9 @@ def is_gemini_image_model(model: str) -> bool: _JSON_VALUE: Final = TypeAdapter(object) +_MODEL_LISTING: Final = TypeAdapter(Mapping[str, object]) +_MODELS: Final = TypeAdapter(Iterable[Mapping[str, object]]) +_MODEL_NAME: Final = TypeAdapter(InstanceOf[str]) def _parse_image_config_string(raw_image_config: str, model: str) -> object: @@ -387,10 +390,10 @@ class GeminiModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model.replace("gemini/", "") - def process_model_name(self, models: list[dict[str, str]]) -> list[str]: + def process_model_name(self, models: Iterable[Mapping[str, object]]) -> list[str]: litellm_model_names: Final = [] for model in models: - stripped_model_name = model["name"].replace("models/", "") + stripped_model_name = _MODEL_NAME.validate_python(model["name"]).replace("models/", "") litellm_model_name = "gemini/" + stripped_model_name litellm_model_names.append(litellm_model_name) return litellm_model_names @@ -414,9 +417,9 @@ class GeminiModelInfo(BaseLLMModelInfo): f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}" ) - models: Final[list[dict[str, str]]] = response.json()["models"] + listing: Final = _MODEL_LISTING.validate_python(response.json()) - litellm_model_names: Final = self.process_model_name(models) + litellm_model_names: Final = self.process_model_name(_MODELS.validate_python(listing["models"])) return litellm_model_names def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index 196a19a141f..45411956629 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -3,9 +3,10 @@ Transformation for Calling Google models in their native format. """ from copy import deepcopy -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast import httpx +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -34,6 +35,16 @@ else: from ..common_utils import get_api_key_from_env +_RESPONSE_BODY: Final = TypeAdapter(dict[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): """ @@ -378,7 +389,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): from litellm.types.google_genai.main import GenerateContentResponse try: - response = raw_response.json() + decoded: Final = _json_body(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming generate content response: {e}", @@ -387,7 +398,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): ) logging_obj.model_call_details["httpx_response"] = raw_response - response = self.convert_citation_sources_to_citations(response) + response: Final = self.convert_citation_sources_to_citations(_RESPONSE_BODY.validate_python(decoded)) return GenerateContentResponse(**response) diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index 31d3963c70c..7fce5c9e36d 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -1,10 +1,11 @@ import base64 -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from io import BufferedReader, BytesIO -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx from httpx._types import RequestFiles +from pydantic import TypeAdapter from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig @@ -32,6 +33,18 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_JSON_VALUES: Final = TypeAdapter(Iterable[object]) +_USAGE_METADATA: Final = TypeAdapter(dict[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" @@ -125,7 +138,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", @@ -133,25 +146,29 @@ class GeminiImageEditConfig(BaseImageEditConfig): headers=raw_response.headers, ) - candidates: Final = response_json.get("candidates", []) + payload: Final = _JSON_OBJECT.validate_python(response_json) + candidates: Final = payload.get("candidates", []) data_list: Final[list[ImageObject]] = [] - for candidate in candidates: - content = candidate.get("content", {}) - parts = content.get("parts", []) - for part in parts: - inline_data = part.get("inlineData") - if inline_data and inline_data.get("data"): + for candidate in _JSON_VALUES.validate_python(candidates): + content = _JSON_OBJECT.validate_python(candidate).get("content", {}) + parts = _JSON_OBJECT.validate_python(content).get("parts", []) + for part in _JSON_VALUES.validate_python(parts): + inline_data = _JSON_OBJECT.validate_python(part).get("inlineData") + image_data = _JSON_OBJECT.validate_python(inline_data).get("data") if inline_data else None + if image_data: data_list.append( ImageObject( - b64_json=inline_data["data"], + b64_json=image_data, url=None, ) ) model_response.data = cast(list[OpenAIImage], data_list) - if "usageMetadata" in response_json: - model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"]) + if "usageMetadata" in payload: + model_response.usage = transform_gemini_image_usage( + _USAGE_METADATA.validate_python(payload["usageMetadata"]) + ) return model_response def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]: diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index d468889b626..fe6b3250db8 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -9,6 +9,7 @@ https://github.com/caozhiyuan/copilot-api """ import os +from collections.abc import Mapping from typing import ( TYPE_CHECKING, Any, @@ -16,6 +17,8 @@ from typing import ( cast, # noqa: TID251 # narrows untyped request dicts at the session boundary ) +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -46,6 +49,9 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_OBJECT_LIST: Final = TypeAdapter(list[object]) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[object, object]) + def github_copilot_supports_responses_api(model: str) -> bool: """ @@ -390,7 +396,9 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ return self._contains_vision_content(input_param) - def _contains_vision_content(self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> bool: + def _contains_vision_content( + self, value: object, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bool: """ Recursively check if a value contains vision content. @@ -407,22 +415,28 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Check arrays if isinstance(value, list): - return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value) + items: Final = _OBJECT_LIST.validate_python(value) + return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in items) # Only check dict/object types if not isinstance(value, dict): return False + fields: Final = _OBJECT_MAPPING.validate_python(value) + # Check if this item is an input_image - item_type: Final = value.get("type") + item_type: Final = fields.get("type") if isinstance(item_type, str) and item_type.lower() == "input_image": return True # Check content field recursively - if "content" in value and isinstance(value["content"], list): - return any( - self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value["content"] - ) + if "content" in fields: + content: Final = fields["content"] + if isinstance(content, list): + content_items: Final = _OBJECT_LIST.validate_python(content) + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in content_items + ) return False diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py index bcebb347aea..7f75a607410 100644 --- a/litellm/llms/groq/cost_calculator.py +++ b/litellm/llms/groq/cost_calculator.py @@ -19,7 +19,10 @@ if TYPE_CHECKING: def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: search_costs: Final = model_info.get("search_context_cost_per_query") cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 - server_tool_use: Final = getattr(usage, "server_tool_use", None) + try: + server_tool_use: Final = usage.server_tool_use + except AttributeError: + return 0.0 if server_tool_use is None: return 0.0 searches: Final = server_tool_use.web_search_requests or 0 diff --git a/litellm/llms/groq/stt/transformation.py b/litellm/llms/groq/stt/transformation.py index 93d36dd7b0f..33e01ab7c0e 100644 --- a/litellm/llms/groq/stt/transformation.py +++ b/litellm/llms/groq/stt/transformation.py @@ -3,6 +3,7 @@ Translate from OpenAI's `/v1/audio/transcriptions` to Groq's `/v1/audio/transcri """ import types +from collections.abc import Mapping from typing import Final import litellm @@ -39,16 +40,17 @@ class GroqSTTConfig: tools: list | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 334c60ee848..cbb34b586e9 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -93,7 +93,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 499cbab3b7b..575820bceee 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -9,10 +9,13 @@ import uuid from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices +_DECODED_JSON: Final = TypeAdapter(object) + class LangGraphSSEStreamIterator: """ @@ -60,7 +63,7 @@ class LangGraphSSEStreamIterator: return None try: - data: Final = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) return self._process_data(data) except json.JSONDecodeError: verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index a740050dd94..9d8b6c06a85 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio """ from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from urllib.parse import quote import httpx @@ -26,6 +26,7 @@ if TYPE_CHECKING: _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_OPAQUE_VALUE: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -122,7 +123,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): return ["lemonade/" + _STR.validate_python(model["id"]) for model in model_list] @staticmethod - def _get_positive_int(value: Any) -> int | None: + def _get_positive_int(value: object) -> int | None: if isinstance(value, bool): return None if isinstance(value, int) and value > 0: @@ -162,7 +163,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): provider_specific_entry.get("context_window"), provider_specific_entry.get("max_context_window"), ): - parsed = self._get_positive_int(value) + parsed = self._get_positive_int(_OPAQUE_VALUE.validate_python(value)) if parsed is not None: return parsed return None diff --git a/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/litellm/llms/litellm_proxy/skills/sandbox_executor.py index 3e38dd81905..8e5c9effa17 100644 --- a/litellm/llms/litellm_proxy/skills/sandbox_executor.py +++ b/litellm/llms/litellm_proxy/skills/sandbox_executor.py @@ -7,7 +7,8 @@ Supports Docker, Podman, and Kubernetes backends. import base64 import os -from typing import Any, Final, Protocol, TypedDict +from collections.abc import Sequence +from typing import Final, Protocol, TypedDict from typing_extensions import ReadOnly @@ -41,6 +42,13 @@ class _GeneratedFile(TypedDict): mime_type: ReadOnly[str] +class _SandboxExecution(TypedDict): + success: ReadOnly[bool] + output: ReadOnly[str] + error: ReadOnly[str] + files: ReadOnly[Sequence[_GeneratedFile]] + + class SkillsSandboxExecutor: """ Executes skill code in llm-sandbox Docker container. @@ -76,7 +84,7 @@ class SkillsSandboxExecutor: code: str, skill_files: dict[str, bytes], requirements: str | None = None, - ) -> dict[str, Any]: + ) -> _SandboxExecution: """ Execute code with skill files in sandbox. diff --git a/litellm/llms/lm_studio/embed/transformation.py b/litellm/llms/lm_studio/embed/transformation.py index 66b2dbb534d..d6264ceb6e0 100644 --- a/litellm/llms/lm_studio/embed/transformation.py +++ b/litellm/llms/lm_studio/embed/transformation.py @@ -7,6 +7,7 @@ Docs - https://lmstudio.ai/docs/basics/server """ import types +from collections.abc import Mapping from typing import Final @@ -18,16 +19,17 @@ class LmStudioEmbeddingConfig: def __init__( self, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/nvidia_nim/embed.py b/litellm/llms/nvidia_nim/embed.py index 5a0105a3776..d2ef446299e 100644 --- a/litellm/llms/nvidia_nim/embed.py +++ b/litellm/llms/nvidia_nim/embed.py @@ -9,6 +9,7 @@ API calling is done using the OpenAI SDK with an api_base """ import types +from collections.abc import Mapping from typing import Final @@ -32,16 +33,17 @@ class NvidiaNimEmbeddingConfig: input_type: str | None = None, truncate: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e62e5e63642..424c275dacc 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -188,6 +188,7 @@ def _normalize_tool_choice(selected_params: dict) -> None: _TOOL_CHOICE_MODES: Final = frozenset({"auto", "none", "required", "any"}) _TOOLS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) _JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def _as_json_object(value: object) -> Mapping[str, object] | None: @@ -700,7 +701,7 @@ class OCIChatConfig(BaseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) if not isinstance(response_json, dict): raise OCIError( diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 284a6ffea62..f7291bbcffe 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -12,6 +12,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class OllamaResponseIdNormalizer: @@ -151,7 +152,7 @@ class OllamaModelInfo(BaseLLMModelInfo): try: resp: Final = httpx.get(f"{base}/api/tags", headers=headers) resp.raise_for_status() - data: Final = resp.json() + data: Final = _DECODED_JSON.validate_python(resp.json()) # Expecting a dict with a 'models' list models_list = [] if isinstance(data, dict) and "models" in data and isinstance(data["models"], list): diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index e472661efb9..c31dc283c50 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -23,13 +23,14 @@ from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.completion_usage import CompletionUsage -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter if TYPE_CHECKING: from aiohttp import ClientSession import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -134,6 +135,16 @@ class OpenAIError(BaseLLMException): ) +_DECODED_JSON: Final = TypeAdapter(object) + + +def decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + ####### Error Handling Utils for OpenAI API ####################### ################################################################### def drop_params_from_unprocessable_entity_error( @@ -152,7 +163,7 @@ def drop_params_from_unprocessable_entity_error( """ invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): - error_json: Final = e.response.json() + error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) error_message: Final = error_json.get("error", {}) error_body = error_message else: diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index 6f911e9c40f..991353295c2 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -9,6 +9,7 @@ from collections.abc import Mapping, Sequence from typing import Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -18,6 +19,8 @@ from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, ) +_DECODED_JSON: Final = TypeAdapter(object) + class OpenAICountTokensHandler(OpenAICountTokensConfig): """ @@ -34,7 +37,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): tools: Sequence[Mapping[str, object]] | None = None, instructions: str | None = None, text: Mapping[str, object] | None = None, - ) -> dict[str, object]: + ) -> object: """ Handle a token counting request to OpenAI's Responses API. @@ -84,7 +87,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): message=error_text, ) - openai_response: Final = response.json() + openai_response: Final = _DECODED_JSON.validate_python(response.json()) verbose_logger.debug("OpenAI response: %s", openai_response) return openai_response diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index b12fdb95765..18e90c414da 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -16,7 +16,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter -from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials +from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials from litellm.llms.openai.responses.count_tokens.handler import ( OpenAICountTokensHandler, ) @@ -197,7 +197,7 @@ class OpenAITokenCounter(BaseTokenCounter): if result is not None: return TokenCountResponse.model_validate( { - "total_tokens": result.get("input_tokens", 0), + "total_tokens": decoded_json_object(result).get("input_tokens", 0), "request_model": request_model, "model_used": model_to_use, "tokenizer_type": "openai_api", diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index ef501720e2d..c9ae5688f94 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import litellm from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload @@ -40,7 +40,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, @@ -150,7 +150,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py index 13c9d805638..8316e17e4d3 100644 --- a/litellm/llms/opensandbox/sandbox/transformation.py +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -20,6 +20,7 @@ from litellm.constants import ( OPEN_SANDBOX_POLL_INTERVAL, OPEN_SANDBOX_READY_TIMEOUT, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.sandbox.transformation import ( SANDBOX_MAX_OUTPUT_BYTES, @@ -43,6 +44,13 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_VALUE: Final = TypeAdapter(object) +def _decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + class OpenSandboxSandboxConfig(BaseSandboxConfig): def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler: if client is not None: @@ -320,7 +328,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): headers=headers, params={"use_server_proxy": use_server_proxy}, ) - data: Final = response.json() + data: Final = _decoded_json_object(_JSON_VALUE.validate_python(response.json())) endpoint: Final = data.get("endpoint") if not endpoint: raise ValueError(f"OpenSandbox did not return an execd endpoint for {sandbox_id}") @@ -491,7 +499,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): if not data: return None try: - parsed: Final = json.loads(data) + parsed: Final = _JSON_VALUE.validate_python(json.loads(data)) except json.JSONDecodeError: return None if not isinstance(parsed, dict): diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 91d40d8247f..617f6d3d76f 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -12,6 +12,7 @@ Ref: https://docs.perplexity.ai/api-reference/responses-post from typing import Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -21,6 +22,8 @@ from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +_DECODED_JSON: Final = TypeAdapter(object) + class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def get_supported_openai_params(self, model: str) -> list: @@ -103,7 +106,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): ) -> ResponsesAPIResponse: """Check for Perplexity's status:'failed' on HTTP 200 before delegating to base.""" try: - raw_response_json = raw_response.json() + raw_response_json = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raw_response_json = None diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index ddc1b9474ac..ece75c6b595 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -40,6 +40,11 @@ class _RunwayMLTask(TypedDict, total=False): failureCode: ReadOnly[str] +def _parse_runway_task(response: httpx.Response) -> _RunwayMLTask: + task: Final[_RunwayMLTask] = response.json() + return task + + class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): """ Configuration for RunwayML image generation models. @@ -238,7 +243,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -287,7 +292,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -333,7 +338,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): } """ try: - response_data: _RunwayMLTask = raw_response.json() + response_data: _RunwayMLTask = _parse_runway_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", @@ -363,7 +368,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete, transforming to OpenAI format") @@ -393,7 +398,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): We need to poll the task until it completes (status SUCCEEDED) using async polling. """ try: - response_data: _RunwayMLTask = raw_response.json() + response_data: _RunwayMLTask = _parse_runway_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", @@ -423,7 +428,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete (async), transforming to OpenAI format") diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index d41ffea17dc..15e76cc4745 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -40,6 +40,11 @@ class _RunwayTtsTaskResponse(TypedDict, total=False): failureCode: ReadOnly[str] +def _parse_runway_tts_task(response: httpx.Response) -> _RunwayTtsTaskResponse: + task: Final[_RunwayTtsTaskResponse] = response.json() + return task + + class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ Configuration for RunwayML Text-to-Speech @@ -323,7 +328,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -371,7 +376,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -462,7 +467,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -492,7 +497,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete, downloading audio") @@ -531,7 +536,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -561,7 +566,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio") diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 8b765b1390b..1908f82d004 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -119,7 +119,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): @staticmethod def _parse_task_response(raw_response: httpx.Response) -> _RunwayTaskResponse: - return raw_response.json() + return _parse_runway_task_response(raw_response) def get_supported_openai_params(self, model: str) -> list: """ diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 14f0e3ead62..eaa805e7769 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -1,3 +1,4 @@ +import builtins import codecs import functools import json @@ -34,6 +35,14 @@ class _SageMakerMessagesChunkData(TypedDict, total=False): choices: ReadOnly[Required[list[Mapping[str, object]]]] +class _SageMakerPayloadPart(TypedDict, total=False): + bytes: ReadOnly[builtins.bytes] + + +class _SageMakerStreamEvent(TypedDict, total=False): + chunk: ReadOnly[_SageMakerPayloadPart] + + class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) @@ -345,7 +354,7 @@ class AWSEventStreamDecoder: ), ) response_dict: Final = event.to_response_dict() - parsed_response: Final = self.parser.parse(response_dict, response_stream_shape) + parsed_response: Final[_SageMakerStreamEvent] = self.parser.parse(response_dict, response_stream_shape) body: Final = response_dict.get("body") if response_dict["status_code"] != 200: diff --git a/litellm/llms/sagemaker/embedding/cohere_transformation.py b/litellm/llms/sagemaker/embedding/cohere_transformation.py index 35352f65dda..8c0f4437edf 100644 --- a/litellm/llms/sagemaker/embedding/cohere_transformation.py +++ b/litellm/llms/sagemaker/embedding/cohere_transformation.py @@ -10,7 +10,8 @@ be of type Object`. Reference: https://docs.cohere.com/v2/reference/embed """ -from typing import TYPE_CHECKING, Any, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -25,6 +26,7 @@ from litellm.llms.bedrock.embed.cohere_transformation import ( BedrockCohereEmbeddingConfig, ) from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse from ..common_utils import SagemakerError @@ -123,7 +125,7 @@ class SagemakerCohereEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index 1395f9b98ec..2cd02683167 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -4,7 +4,8 @@ Translate from OpenAI's `/v1/embeddings` to Sagemaker's `/invoke` In the Huggingface TGI format. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Sequence +from typing import TYPE_CHECKING, Final if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15,6 +16,7 @@ from httpx._models import Headers, Response from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage from ..common_utils import SagemakerError @@ -158,7 +160,7 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index a82dfe72322..50ab84105d6 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator from .handler import ( AsyncSAPStreamIterator, GenAIHubOrchestrationError, @@ -185,11 +185,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): # Keep a short, tight client lifecycle here to avoid fd leaks client: Final = litellm.module_level_client # with httpx.Client(timeout=30) as client: - deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + f"{self.base_url}/lm/deployments", headers=self.headers + ).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): if dep.get("scenarioId") == "orchestration": - cfg = client.get( + cfg: AICoreConfiguration = client.get( f"{self.base_url}/lm/configurations/{dep['configurationId']}", headers=self.headers, ).json() diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index b1618f38623..705ea843e64 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -33,6 +33,21 @@ DEFAULT_HOME_PATH: Final = os.path.join(os.path.expanduser("~"), ".aicore") _DECODED_JSON: Final = TypeAdapter(object) +class AICoreDeployment(TypedDict): + configurationId: ReadOnly[str] + scenarioId: ReadOnly[str] + deploymentUrl: ReadOnly[str] + createdAt: ReadOnly[str] + + +class AICoreDeploymentList(TypedDict, total=False): + resources: ReadOnly[Sequence[AICoreDeployment]] + + +class AICoreConfiguration(TypedDict): + executableId: ReadOnly[str] + + def _get_home() -> str: return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH) @@ -195,7 +210,7 @@ def resolve_resource_group(sources: list[Source]) -> str | None: def _parse_service_key_once( service_key: str | dict | None, -) -> dict[str, object] | None: +) -> object: """ Pre-parse service_key if it's a string to avoid repeated JSON parsing. @@ -207,7 +222,7 @@ def _parse_service_key_once( return service_key if isinstance(service_key, str): try: - return json.loads(service_key) + return _DECODED_JSON.validate_python(json.loads(service_key)) except json.JSONDecodeError: verbose_logger.warning("SAP service key is a string but not valid JSON. Skipping this source.") return None @@ -217,7 +232,7 @@ def _parse_service_key_once( return None -def _resolve_credential_from_service_key(service_key: str | dict | None, cv: CredentialsValue) -> str | None: +def _resolve_credential_from_service_key(service_key: object, cv: CredentialsValue) -> str | None: if service_key is None: return None val: Final = _str_or_none(_get_nested(service_key, (("credentials",) + cv.vcap_key) if cv.vcap_key else (cv.name,))) @@ -252,14 +267,16 @@ def fetch_credentials( """ config: Final = init_conf(profile) - service_key = _parse_service_key_once(service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR)) + parsed_service_key: Final = _parse_service_key_once( + service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR) + ) vcap_service: Final = _get_vcap_service(VCAP_AICORE_SERVICE_NAME) sources: Final = [ Source("kwargs", lambda cv: _str_or_none(kwargs.get(cv.name))), Source( "service key", - lambda cv: _resolve_credential_from_service_key(service_key, cv), + lambda cv: _resolve_credential_from_service_key(parsed_service_key, cv), ), Source( "environment variables", diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index e91dfb91081..17e96217a2c 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -18,7 +18,7 @@ from litellm.types.llms.openai import AllEmbeddingInputValues from litellm.types.utils import EmbeddingResponse from ..chat.handler import GenAIHubOrchestrationError -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator class Usage(LiteLLMBaseModel): @@ -101,10 +101,12 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): def deployment_url(self) -> str: with httpx.Client(timeout=30) as client: valid_deployments: Final = [] - deployments: Final = client.get(self.base_url + "/lm/deployments", headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + self.base_url + "/lm/deployments", headers=self.headers + ).json() for deployment in deployments.get("resources", []): if deployment["scenarioId"] == "orchestration": - config_details = client.get( + config_details: AICoreConfiguration = client.get( self.base_url + f"/lm/configurations/{deployment['configurationId']}", headers=self.headers, ).json() diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 5d5fad6128e..974996c5e48 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -6,10 +6,12 @@ Handles transformation between OpenAI-compatible format and Stability AI API for API Reference: https://platform.stability.ai/docs/api-reference """ +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles +from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -30,6 +32,13 @@ else: LiteLLMLoggingObj = Any +class _StabilityImageEditResponse(TypedDict, total=False): + image: ReadOnly[str] + finish_reason: ReadOnly[str] + seed: ReadOnly[int] + errors: ReadOnly[Sequence[str]] + + class StabilityImageEditConfig(BaseImageEditConfig): """ Configuration for Stability AI image edit. @@ -259,7 +268,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp} """ try: - response_data: Final = raw_response.json() + response_data: Final[_StabilityImageEditResponse] = raw_response.json() except Exception as e: raise self.get_error_class( error_message=f"Error parsing Stability AI response: {e}", diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index b37bbd78f2b..fe0f96cb805 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Final, Literal from httpx import Headers, Response +from typing_extensions import ReadOnly, TypedDict from litellm.constants import DEFAULT_MAX_TOKENS_FOR_TRITON from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory @@ -32,6 +33,10 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +class _TritonGenerateResponse(TypedDict): + text_output: ReadOnly[str] + + class TritonConfig(BaseConfig): """ Base class for Triton configurations. @@ -220,7 +225,7 @@ class TritonGenerateConfig(TritonConfig): json_mode: bool | None = None, ) -> ModelResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonGenerateResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) model_response.choices = [Choices(index=0, message=Message(content=raw_response_json["text_output"]))] diff --git a/litellm/llms/triton/embedding/transformation.py b/litellm/llms/triton/embedding/transformation.py index 8c6e0b832fc..3a735f741fd 100644 --- a/litellm/llms/triton/embedding/transformation.py +++ b/litellm/llms/triton/embedding/transformation.py @@ -1,6 +1,8 @@ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException from litellm.llms.base_llm.embedding.transformation import ( @@ -14,6 +16,16 @@ from litellm.utils import token_counter from ..common_utils import TritonError +class _TritonEmbeddingOutput(TypedDict): + shape: ReadOnly[Sequence[int]] + data: ReadOnly[Sequence[float]] + + +class _TritonEmbeddingResponse(TypedDict): + outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]] + model_name: NotRequired[ReadOnly[str]] + + class TritonEmbeddingConfig(BaseEmbeddingConfig): """ Transformations for triton /embeddings endpoint (This is a trtllm model) @@ -79,7 +91,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict = {}, ) -> EmbeddingResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonEmbeddingResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) @@ -134,7 +146,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): return TritonError(message=error_message, status_code=status_code, headers=headers) @staticmethod - def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]: + def split_embedding_by_shape(data: Sequence[float], shape: Sequence[int]) -> list[Sequence[float]]: if len(shape) != 2: raise ValueError("Shape must be of length 2.") embedding_size: Final = shape[1] diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index 19c4c2659ca..b84abacbdaf 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -6,9 +6,11 @@ Calls done in OpenAI/openai.py as Vercel AI Gateway is openai-compatible. Docs: https://vercel.com/docs/ai-gateway """ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -19,6 +21,14 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import VercelAIGatewayException +class _VercelAIGatewayModel(TypedDict): + id: ReadOnly[str] + + +class _VercelAIGatewayModelList(TypedDict): + data: ReadOnly[Sequence[_VercelAIGatewayModel]] + + class VercelAIGatewayConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> str | None: @@ -98,5 +108,5 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): if response.status_code != 200: raise Exception(f"Failed to get models: {response.text}") - models: Final = response.json()["data"] - return [model["id"] for model in models] + models: Final[_VercelAIGatewayModelList] = response.json() + return [model["id"] for model in models["data"]] diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index a8d02045740..99ef750b5fc 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -202,7 +202,7 @@ class VertexAIBatchTransformation: @classmethod def transform_vertex_ai_batch_list_response_to_openai_list_response( cls, response: dict[str, Any] - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Transforms Vertex AI batch list response into OpenAI-compatible list response. """ diff --git a/litellm/llms/vertex_ai/files/handler.py b/litellm/llms/vertex_ai/files/handler.py index f2da04a7db7..54c0fa97550 100644 --- a/litellm/llms/vertex_ai/files/handler.py +++ b/litellm/llms/vertex_ai/files/handler.py @@ -3,7 +3,7 @@ import json import os import time from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final from urllib.parse import unquote import httpx @@ -197,7 +197,7 @@ class VertexAIFilesHandler(GCSBucketBase): timeout: float | httpx.Timeout, max_retries: int | None, litellm_params: dict | None = None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from GCS bucket for VertexAI files. Supports both sync and async operations. diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 396acd270fa..2d560216671 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -5,7 +5,7 @@ import json import os import re import time -from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping +from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping, Sequence from contextlib import aclosing from dataclasses import dataclass from types import MappingProxyType @@ -15,6 +15,7 @@ from urllib.parse import quote, unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted +from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required import litellm @@ -30,6 +31,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, @@ -93,6 +95,7 @@ _JSONL_NEWLINE: Final = b"\n" _BATCH_OUTPUT_FIRST_ROW_PEEK_LIMIT_BYTES: Final = 32 * 1024 * 1024 _PASSTHROUGH_MANAGED_GCS_PREFIX: Final = f"{VERTEX_AI_MANAGED_GCS_PREFIX}passthrough/" _RAW_UPLOAD_CHUNK_BYTES: Final = 1024 * 1024 +_DECODED_JSON: Final = TypeAdapter(object) class _GcsObjectMetadataJson(TypedDict, total=False): @@ -824,7 +827,7 @@ def _iter_openai_jsonl_entries( def _load_openai_jsonl_row(lineno: int, line: str) -> object: try: - return json.loads(line) + return _DECODED_JSON.validate_python(json.loads(line)) except json.JSONDecodeError as e: raise VertexAIError( status_code=400, @@ -964,7 +967,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def _get_gcs_object_name_from_batch_jsonl( self, - openai_jsonl_content: list[dict[str, Any]], + openai_jsonl_content: Sequence[Mapping[str, object]], deployment_model: str | None = None, ) -> str: """ @@ -982,8 +985,9 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): `projects/../locations/../endpoints/` batch job model instead of a nonexistent publisher model. """ - first_entry: Final = openai_jsonl_content[0] - body_model: Final = first_entry["body"].get("model") if isinstance(first_entry.get("body"), dict) else None + body: Final = openai_jsonl_content[0].get("body") + body_fields: Final = as_str_mapping(body) + body_model: Final = body_fields.get("model") if body_fields is not None and isinstance(body, dict) else None jsonl_model: Final = body_model if isinstance(body_model, str) else "" raw_model: Final = deployment_model.removeprefix("vertex_ai/") if deployment_model else jsonl_model return _managed_batch_object_name(raw_model, passthrough=False) diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 40564d536a7..7cb52281a6a 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -27,6 +27,11 @@ from litellm.types.utils import LiteLLMFineTuningJob _JSON_VALUE: Final = TypeAdapter(object) +def _response_tuning_job(response: httpx.Response) -> ResponseTuningJob: + tuning_job: Final[ResponseTuningJob] = response.json() + return ResponseTuningJob(**tuning_job) + + class VertexFineTuningAPI(VertexLLM): """ Vertex methods to support for batches @@ -187,9 +192,7 @@ class VertexFineTuningAPI(VertexLLM): "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) @@ -272,9 +275,7 @@ class VertexFineTuningAPI(VertexLLM): verbose_logger.debug( "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) diff --git a/litellm/llms/vertex_ai/gemini/long_decode.py b/litellm/llms/vertex_ai/gemini/long_decode.py index 81d6c04d0e8..ef013d95088 100644 --- a/litellm/llms/vertex_ai/gemini/long_decode.py +++ b/litellm/llms/vertex_ai/gemini/long_decode.py @@ -285,7 +285,8 @@ def with_initial_prompt_tokens(usage: Usage, initial_prompt_tokens: int | None) def parse_body(response: httpx.Response) -> GenerateContentResponseBody: try: - return GenerateContentResponseBody(**response.json()) + body: Final[GenerateContentResponseBody] = response.json() + return GenerateContentResponseBody(**body) except Exception as e: raise VertexAIError( message=( diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index 0a800497de1..f15c6393cb2 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -6,6 +6,7 @@ import json from typing import TYPE_CHECKING, Final, Literal import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.llms.custom_httpx.http_handler import ( @@ -35,6 +36,11 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +class _GeminiFileResource(TypedDict, total=False): + mimeType: ReadOnly[str] + uri: ReadOnly[str] + + class GoogleBatchEmbeddings(VertexLLM): @staticmethod def _flatten_and_detect_file_refs( @@ -74,7 +80,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), @@ -111,7 +117,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index f916d494aad..43d3e23855e 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles from pydantic import ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -33,6 +34,10 @@ _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object] ) +class _VertexImagenPredictResponse(TypedDict, total=False): + predictions: ReadOnly[Sequence[object]] + + class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): """ Vertex AI Imagen Image Edit Configuration @@ -206,7 +211,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final[_VertexImagenPredictResponse] = raw_response.json() except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", diff --git a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py index 7ba7b060991..ec915e10fcd 100644 --- a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py @@ -168,7 +168,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) async def aimage_generation( @@ -272,7 +272,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) def is_image_generation_response(self, json_response: dict[str, Any]) -> bool: diff --git a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py index da77fa338db..c282ae22ea6 100644 --- a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py @@ -208,7 +208,7 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig): if raw_response.status_code != 200: raise Exception(f"Error: {raw_response.status_code} {raw_response.text}") - _json_response: Final = raw_response.json() + _json_response: Final[MultimodalPredictions] = raw_response.json() if "predictions" not in _json_response: raise InternalServerError( message=f"embedding response does not contain 'predictions', got {_json_response}", diff --git a/litellm/llms/vertex_ai/rag_engine/transformation.py b/litellm/llms/vertex_ai/rag_engine/transformation.py index a34ef519578..61cb083fa4c 100644 --- a/litellm/llms/vertex_ai/rag_engine/transformation.py +++ b/litellm/llms/vertex_ai/rag_engine/transformation.py @@ -4,7 +4,10 @@ Transformation utilities for Vertex AI RAG Engine. Handles transforming LiteLLM's unified formats to Vertex AI RAG Engine API format. """ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final + +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE @@ -13,6 +16,15 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.rag import RAGChunkingStrategy +class VertexRAGChunkingConfig(TypedDict): + chunk_size: ReadOnly[int] + chunk_overlap: ReadOnly[int] + + +class VertexRAGTransformationConfig(TypedDict): + chunking_config: ReadOnly[VertexRAGChunkingConfig] + + class VertexAIRAGTransformation(VertexBase): """ Transformation class for Vertex AI RAG Engine API. @@ -55,7 +67,7 @@ class VertexAIRAGTransformation(VertexBase): def transform_chunking_strategy_to_vertex_format( self, chunking_strategy: RAGChunkingStrategy | None, - ) -> dict[str, Any]: + ) -> VertexRAGTransformationConfig: """ Transform LiteLLM's unified chunking_strategy to Vertex AI RAG format. @@ -105,7 +117,7 @@ class VertexAIRAGTransformation(VertexBase): self, gcs_uri: str, chunking_strategy: RAGChunkingStrategy | None = None, - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Build the request payload for importing RAG files. diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index dce4d2f2a87..398471a7c9b 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -6,10 +6,11 @@ Why separate file? Make it easy to see how transformation works import math import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -27,6 +28,17 @@ from litellm.types.rerank import ( ) +class _VertexRankedRecord(TypedDict): + id: ReadOnly[str] + score: NotRequired[ReadOnly[float]] + title: NotRequired[ReadOnly[str]] + content: NotRequired[ReadOnly[str]] + + +class _VertexRankResponse(TypedDict, total=False): + records: ReadOnly[Sequence[_VertexRankedRecord]] + + class VertexAIRerankConfig(BaseRerankConfig, VertexBase): """ Configuration for Vertex AI Discovery Engine Rerank API @@ -172,7 +184,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): Transform Vertex AI Discovery Engine response to Cohere format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_VertexRankResponse] = raw_response.json() except Exception as e: raise ValueError(f"Failed to parse response: {e}") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py index 45cc5320401..3873a4be0b5 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final import litellm @@ -18,16 +19,17 @@ class VertexAIAi21Config(OpenAIGPTConfig): self, max_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index ddc83bc53a5..96fccf007a5 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final, Literal from pydantic import TypeAdapter @@ -70,16 +71,17 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): | None = None, title: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 6e0aa88b6b6..002bcba2c3a 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -45,6 +45,11 @@ class _EventModelClass(Protocol): def model_validate(self, obj: Mapping[str, object]) -> ResponsesAPIStreamingResponse: ... +class _FieldDefaultCarrier(Protocol): + @property + def default(self) -> object: ... + + class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): _SUPPORTED_OPTIONAL_PARAMS: list[str] = [ # Doc-listed knobs @@ -430,6 +435,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): annotation: Final[object] = field.annotation return annotation + @staticmethod + def _field_default(field: _FieldDefaultCarrier) -> object: + return field.default + @staticmethod def _fill_missing_fields(chunk: Mapping[str, object], event_model: object | None) -> Mapping[str, object]: """ @@ -451,7 +460,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): continue # Explicit default or factory - field_default: object = field.default + field_default: object = VolcEngineResponsesAPIConfig._field_default(field) if field_default is not pyd_fields.PydanticUndefined and field_default is not None: patched[name] = field_default continue diff --git a/litellm/main.py b/litellm/main.py index 71cb7af1fbb..e9c1142167f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4197,6 +4197,7 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( @@ -4741,7 +4742,7 @@ def _complete_petals(ctx: CompletionDispatchContext) -> _CompletionDispatchResul ) if stream is True: ## [BETA] # Fake streaming for petals - resp_string: Final = model_response["choices"][0]["message"]["content"] + resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"]) return CustomStreamWrapper( resp_string, model, @@ -7886,7 +7887,6 @@ def text_completion( tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): - responses: Final = [None for x in prompt] # init responses def process_prompt(i, individual_prompt): decoded_prompt: Final = tokenizer.decode(individual_prompt) @@ -7903,16 +7903,16 @@ def text_completion( text_completion_response["object"] = "text_completion" text_completion_response["created"] = response.get("created", None) text_completion_response["model"] = response.get("model", None) - return response["choices"][0] + return response.choices[0] with concurrent.futures.ThreadPoolExecutor() as executor: completed_futures: Final = [ executor.submit(process_prompt, i, individual_prompt) for i, individual_prompt in enumerate(prompt) ] - for i, future in enumerate(concurrent.futures.as_completed(completed_futures)): - responses[i] = future.result() - text_completion_response.choices = responses + text_completion_response.choices = [ + future.result() for future in concurrent.futures.as_completed(completed_futures) + ] return text_completion_response # else: diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 7302b74f15a..b23d1094962 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -165,6 +165,7 @@ _CLIENT_FORWARDED_AUTH_TYPES: Final["frozenset[str]"] = frozenset({"true_passthr _MINTED_TOKEN_CREDENTIAL_FIELDS: Final["frozenset[str]"] = frozenset({"access_token", "refresh_token", "expires_in"}) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) def _bind_submitted_oauth_client( @@ -339,7 +340,7 @@ def _decrypt_env_vars_on_returned_row(row: object) -> None: return if isinstance(env_vars, str): try: - env_vars = json.loads(env_vars) + env_vars = _DECODED_JSON.validate_python(json.loads(env_vars)) except (json.JSONDecodeError, TypeError): return if not isinstance(env_vars, list): diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 3b2c2e09df5..f5c54451006 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -701,7 +701,7 @@ async def _store_per_user_token_server_side( expires_in = None refresh_token: Final[str | None] = token_response.get("refresh_token") or None - raw_scope: Final[object] = token_response.get("scope", requested_scope) + raw_scope: Final[object] = _DECODED_JSON.validate_python(token_response.get("scope", requested_scope)) scopes: Final[list[str] | None] = raw_scope.split() if isinstance(raw_scope, str) else None try: diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index f65310a77e3..567fb384750 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -488,6 +488,10 @@ class _AuthorizationServerMetadataPayload(TypedDict, total=False): token_endpoint_auth_methods_supported: Sequence[str] +def _mapping_value(source: Mapping[str, object], key: str, default: object) -> object: + return source.get(key, default) + + def _blank_to_none(value: str | None) -> str | None: """Collapse an absent, empty, or whitespace-only string to ``None``. @@ -2718,7 +2722,9 @@ class MCPServerManager: server_id=server_id, name=name_for_prefix, protocol_version=TypeAdapter(MCPUpstreamProtocol).validate_python( - server_config.get("protocol_version", mcp_info.get("protocol_version", "auto")) + _mapping_value( + server_config, "protocol_version", _mapping_value(mcp_info, "protocol_version", "auto") + ) ), alias=alias, server_name=server_name, diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 6d4fefcdb6c..cc92b8cce9b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -41,6 +42,8 @@ from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer +_DECODED_JSON: Final = TypeAdapter(object) + class MCPOAuth2TokenCache(InMemoryCache): """ @@ -160,7 +163,7 @@ class MCPOAuth2TokenCache(InMemoryCache): f"failed with status {exc.response.status_code}" ) from exc - body: Final = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index c31560a9f63..b3aa040c507 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -18,6 +18,7 @@ from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException from jwt.types import Options +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -38,6 +39,7 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( "PS512", ) _JWKS_CACHE_TTL_SECONDS: Final = 3600 +_DECODED_JSON: Final = TypeAdapter(object) _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher: TypeAlias = Callable[ @@ -103,7 +105,7 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(jwks_url) response.raise_for_status() - document: Final = response.json() + document: Final = _DECODED_JSON.validate_python(response.json()) keys: Final = document.get("keys") if isinstance(document, dict) else None if not isinstance(keys, list): raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") @@ -116,7 +118,7 @@ async def _discover_jwks_url(issuer: str) -> str: client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(discovery_url) response.raise_for_status() - metadata: Final = response.json() + metadata: Final = _DECODED_JSON.validate_python(response.json()) jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None if not isinstance(jwks_uri, str) or not jwks_uri: raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") diff --git a/litellm/proxy/_experimental/mcp_server/result_conversion.py b/litellm/proxy/_experimental/mcp_server/result_conversion.py index 61435f94c94..fdb9a01fea6 100644 --- a/litellm/proxy/_experimental/mcp_server/result_conversion.py +++ b/litellm/proxy/_experimental/mcp_server/result_conversion.py @@ -13,7 +13,7 @@ from __future__ import annotations import json import math from collections.abc import Sequence -from typing import Final, TypeAlias, TypeVar +from typing import Final, Protocol, TypeAlias, TypeVar from mcp.types import CacheableResult, CallToolResult, ContentBlock, InputRequiredResult, TextContent, Tool from typing_extensions import ReadOnly, TypedDict, assert_never @@ -105,8 +105,17 @@ def complete_call_tool_result(outcome: ToolOutcome, compat: WireCompat) -> CallT return converted +class _StructuredContentCarrier(Protocol): + @property + def structured_content(self) -> object: ... + + +def _structured_content(result: _StructuredContentCarrier) -> object: + return result.structured_content + + def _downgrade_structured_content(result: CallToolResult) -> CallToolResult: - structured: Final = result.structured_content + structured: Final = _structured_content(result) if structured is None or isinstance(structured, dict): return result fallback: Final = TextContent(type="text", text=json.dumps(structured)) diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index 2f88d8c361f..a7748b9ca88 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -6,7 +6,7 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. import asyncio from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, TypeVar from litellm._logging import verbose_logger from litellm.exceptions import ContextWindowExceededError @@ -20,6 +20,8 @@ if TYPE_CHECKING: from litellm.router import Router +_Tool: Final = TypeVar("_Tool") + class SemanticToolFilterContextWindowError(Exception): """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" @@ -265,9 +267,9 @@ class SemanticMCPToolFilter: async def filter_tools( self, query: str, - available_tools: list[Any], + available_tools: list[_Tool], top_k: int | None = None, - ) -> list[Any]: + ) -> list[_Tool]: """ Filter tools semantically based on query. @@ -390,7 +392,7 @@ class SemanticMCPToolFilter: separator: Final = client_name[-len(canonical) - 1] return separator in ("_", "-") - def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]: + def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_Tool]) -> list[_Tool]: """ Get tools from available_tools by their names, preserving the semantic router's ordering. @@ -406,13 +408,13 @@ class SemanticMCPToolFilter: # Exact matches win over suffix matches when both are present, and # each incoming tool is returned at most once even if two canonical # names happen to be tail-compatible with the same incoming name. - available_by_name: Final[dict[str, object]] = {} + available_by_name: Final[dict[str, _Tool]] = {} for tool in available_tools: client_name, _ = self.extract_tool_info(tool) if client_name and client_name not in available_by_name: available_by_name[client_name] = tool - matched: Final[list[object]] = [] + matched: Final[list[_Tool]] = [] used_ids: Final[set[int]] = set() for canonical in tool_names: tool = available_by_name.get(canonical) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index b3426089375..74880bddcd9 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -14,7 +14,7 @@ import time import types from collections import Counter from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence, Set -from typing import TYPE_CHECKING, Final, NoReturn, Protocol +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol import httpx from fastapi import FastAPI, HTTPException @@ -150,6 +150,16 @@ def _scope_headers(scope: Scope) -> Sequence[tuple[bytes, bytes]]: return asgi_scope_fields(scope).get("headers", []) +class _RawHeaderPairsSource(Protocol): + def get( + self, key: Literal["headers"], default: Sequence[tuple[bytes | str, bytes | str]], / + ) -> Sequence[tuple[bytes | str, bytes | str]]: ... + + +def _raw_header_pairs(source: _RawHeaderPairsSource) -> Sequence[tuple[bytes | str, bytes | str]]: + return source.get("headers", []) + + def unsupported_protocol_version(scope: Scope) -> str | None: """Admit configured HTTP revisions while keeping SSE on the legacy protocol.""" from litellm.proxy._experimental.mcp_server.capabilities import configured_versions @@ -1319,7 +1329,7 @@ if MCP_AVAILABLE: Extract mcp-session-id from ASGI scope headers. Returns None if not present. """ - scope_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + scope_headers: Final = _raw_header_pairs(scope) for header_name, header_value in scope_headers: name = header_name if isinstance(header_name, bytes) else header_name.encode() if name.lower() == b"mcp-session-id": @@ -1669,7 +1679,7 @@ if MCP_AVAILABLE: Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header: Final = b"mcp-session-id" - _headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + _headers: Final = _raw_header_pairs(scope) def _normalize_header_name(header_name: object) -> bytes | None: if isinstance(header_name, bytes): @@ -2869,7 +2879,7 @@ if MCP_AVAILABLE: mcp_servers, _sse_client_ip, scoped_server_endpoint=scoped_server_endpoint, - is_initialize=scope.get("method") == "GET", + is_initialize=asgi_scope_fields(scope).get("method", "") == "GET", ): async with ( sse.connect_sse(transport_scope, receive, send) as (read_stream, write_stream), @@ -3090,7 +3100,7 @@ if MCP_AVAILABLE: ) -> Send: async def wrapped_send(message: Message) -> None: if message.get("type") == "http.response.start": - response_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = message.get("headers", []) + response_headers: Final = _raw_header_pairs(message) for key, value in response_headers: header_name = key if isinstance(key, bytes) else str(key).encode() if header_name.lower() == b"mcp-session-id": diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 1684b9c5c30..ee8f3ea3b30 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -13,6 +13,8 @@ from collections.abc import Set as AbstractSet from typing import Any, Final, Literal, Protocol, overload from urllib.parse import quote +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -21,6 +23,7 @@ if typing.TYPE_CHECKING: MCP_SERVERS_TARGET: Final = "mcp_servers" MCP_OAUTH_TOKENS_TARGET: Final = "mcp_oauth_tokens" +_DECODED_JSON: Final = TypeAdapter(object) class ASGIScopeFields(Protocol): @@ -243,7 +246,7 @@ def _parse_mcp_info_dict(mcp_info: object) -> Mapping[str, object] | None: return mcp_info if isinstance(mcp_info, str): try: - parsed: Final[object] = json.loads(mcp_info) + parsed: Final = _DECODED_JSON.validate_python(json.loads(mcp_info)) except (ValueError, TypeError): return None return parsed if isinstance(parsed, dict) else None diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index a7adb5c1bf5..a18fa194c9d 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -361,7 +361,7 @@ class LazyFeatureMiddleware: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # Short-circuit once every feature has loaded. if scope["type"] in ("http", "websocket") and len(self._loaded) < len(self._features): - path: str = scope.get("path", "") + path: str = str(_scope_value(scope, "path")) # Strip the request's root_path so prefix matching works under a # server root path. Without this, requests like /api/v1/policies/... # never match the registered prefixes (/policies/...) and lazy @@ -589,6 +589,7 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY from fastapi import APIRouter, Depends, HTTPException from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router: Final = APIRouter() @@ -608,17 +609,22 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY await _force_load(app, feat, features) feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] - full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) + full: Final[Mapping[str, object]] = MappingProxyType( + get_openapi(title=app.title, version=app.version, routes=feat_routes) + ) + paths: Final = as_str_mapping(full.get("paths", {})) or {} + components: Final = as_str_mapping(full.get("components", {})) or {} # Force all operations under one tag so they group under a single Swagger # section — many lazy modules tag routes inconsistently. - for path_ops in full.get("paths", {}).values(): - for op in path_ops.values(): - if isinstance(op, dict): - op["tags"] = [feat.name] + for path_ops in paths.values(): + for op in (as_str_mapping(path_ops) or {}).values(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation["tags"] = [feat.name] return { "stub_path": feat.path_prefixes[0], "paths": full.get("paths", {}), - "components": {"schemas": full.get("components", {}).get("schemas", {})}, + "components": {"schemas": components.get("schemas", {})}, } return router diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 00a05307a89..c5a63b0f990 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -16,6 +16,7 @@ import sys from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, Final from typing_extensions import ReadOnly, TypedDict @@ -61,7 +62,7 @@ def load_snapshot() -> dict[str, dict] | None: return None -def _normalize_operation_ids(paths: dict[str, dict]) -> None: +def _normalize_operation_ids(paths: Mapping[str, object]) -> None: """Make FastAPI-generated operation IDs stable for multi-method routes. FastAPI derives the default operation ID suffix from the first item in the @@ -69,26 +70,30 @@ def _normalize_operation_ids(paths: dict[str, dict]) -> None: set iteration order can vary between processes, which makes the snapshot drift even when no routes changed. """ + from litellm.litellm_core_utils.core_helpers import as_str_mapping + for path_ops in paths.values(): - if not isinstance(path_ops, dict): + operations = as_str_mapping(path_ops) + if not isinstance(operations, dict): continue - methods = {method for method in path_ops if method in HTTP_METHOD_SUFFIXES} + methods = {method for method in operations if method in HTTP_METHOD_SUFFIXES} if not methods: continue - for method, operation in path_ops.items(): - if method not in HTTP_METHOD_SUFFIXES or not isinstance(operation, dict): + for method, operation in operations.items(): + entry = as_str_mapping(operation) + if method not in HTTP_METHOD_SUFFIXES or not isinstance(entry, dict): continue - operation_id = operation.get("operationId") + operation_id = entry.get("operationId") if not isinstance(operation_id, str): continue for suffix in methods: suffix_token = f"_{suffix}" if operation_id.endswith(suffix_token): - operation["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" + entry["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" break @@ -117,6 +122,7 @@ def _register_feature(app: "FastAPI", feat: "LazyFeature") -> str | None: def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: set[str]) -> SnapshotFragment | None: from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.proxy_server import ensure_unique_openapi_operation_ids feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] @@ -124,22 +130,29 @@ def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: s return None _stabilize_multi_method_route_ids(feat_routes) full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) - paths: Final = full.get("paths", {}) + openapi: Final[Mapping[str, object]] = MappingProxyType(full) + paths: Final = as_str_mapping(openapi.get("paths", {})) or {} _normalize_operation_ids(paths) for path_ops in paths.values(): - for method, op in path_ops.items(): - if isinstance(op, dict): - operation_id = op.get("operationId") + for method, op in (as_str_mapping(path_ops) or {}).items(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation_id = operation.get("operationId") if isinstance(operation_id, str): for suffix in HTTP_METHOD_SUFFIXES: if operation_id.endswith(f"_{suffix}"): - op["operationId"] = operation_id[: -len(suffix)] + method + operation["operationId"] = operation_id[: -len(suffix)] + method break - op["tags"] = [feat.name] - unique: Final = ensure_unique_openapi_operation_ids(full, used_operation_ids) + operation["tags"] = [feat.name] + unique: Final[Mapping[str, object]] = MappingProxyType( + ensure_unique_openapi_operation_ids(full, used_operation_ids) + ) + components: Final = as_str_mapping(unique.get("components", {})) or {} return { - "paths": paths, - "components": {"schemas": unique.get("components", {}).get("schemas", {})}, + "paths": { + path: operations for path, entry in paths.items() if (operations := as_str_mapping(entry)) is not None + }, + "components": {"schemas": as_str_mapping(components.get("schemas", {})) or {}}, } diff --git a/litellm/proxy/a2a/discovery.py b/litellm/proxy/a2a/discovery.py index ff58c9c85ec..03d1be7d185 100644 --- a/litellm/proxy/a2a/discovery.py +++ b/litellm/proxy/a2a/discovery.py @@ -16,10 +16,13 @@ fetcher dispatches by ``discovery_mode``: from collections.abc import Mapping from enum import Enum -from typing import Any, Final +from typing import Final from urllib.parse import urlencode +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider @@ -46,6 +49,8 @@ AGENT_CARD_WELL_KNOWN_PATHS: Final[tuple[str, ...]] = ( DEFAULT_DISCOVERY_TIMEOUT_SECONDS: Final = 10.0 +_DECODED_JSON: Final = TypeAdapter(object) + class AgentCardDiscoveryError(Exception): """Raised when none of the well-known paths returned a usable agent card.""" @@ -87,7 +92,7 @@ async def fetch_well_known_card( params: Mapping[str, object] | None = None, timeout: float = DEFAULT_DISCOVERY_TIMEOUT_SECONDS, headers: dict[str, str] | None = None, -) -> dict[str, Any]: +) -> dict[str, object]: """ Fetch an agent card from ``base_url`` using the strategy chosen by ``discovery_mode``. Returns the parsed JSON from the first path that @@ -137,13 +142,14 @@ async def fetch_well_known_card( continue try: - card = response.json() + decoded = _DECODED_JSON.validate_python(response.json()) except Exception as exc: last_error = f"{url}: invalid JSON ({exc})" continue + card = as_str_mapping(decoded) if not isinstance(card, dict): - last_error = f"{url}: expected JSON object, got {type(card).__name__}" + last_error = f"{url}: expected JSON object, got {type(decoded).__name__}" continue verbose_proxy_logger.debug("A2A discovery succeeded at %s", url) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index c11d296837e..4691c7b0bc5 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -23,6 +23,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header from litellm.proxy._types import UserAPIKeyAuth @@ -281,7 +282,7 @@ async def _a2a_sse_event_source( request_id: str | int | None = None, extra_headers: Mapping[str, str] | None = None, served_version: A2AVersion = "0.3", -) -> AsyncGenerator[Mapping[str, object], None]: +) -> AsyncGenerator[object, None]: """Stream an upstream A2A SSE response as parsed JSON-RPC event dicts. Upstream HTTP/JSON-RPC errors are surfaced as a single JSON-RPC error event @@ -327,12 +328,14 @@ async def _a2a_sse_event_source( if not payload: continue try: - event = json.loads(payload) + event = _DECODED_JSON.validate_python(json.loads(payload)) except Exception: continue - if isinstance(event, dict): - event = normalize_stream_event(event, served_version, request_id=request_id) - yield event + event_fields = as_str_mapping(event) + if isinstance(event_fields, dict): + yield normalize_stream_event(event_fields, served_version, request_id=request_id) + else: + yield event finally: await resp.aclose() @@ -741,7 +744,7 @@ async def invoke_agent_a2a( # Get backend URL and agent name agent_card_params: Final = agent.agent_card_params or {} agent_url: Final = agent_card_params.get("url") - agent_name: Final = agent_card_params.get("name", agent_id) + agent_name: Final = (as_str_mapping(agent_card_params) or {}).get("name", agent_id) # Get litellm_params (may include custom_llm_provider for completion bridge) litellm_params: dict[str, object] = agent.litellm_params or {} diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 9a6f674db2c..fb727367caf 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -18,6 +18,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.handler import ( bridge_model_name, ) from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints.databricks_oauth import ( resolve_databricks_backend_auth, @@ -173,13 +174,14 @@ async def route_a2a_agent_request( return _call_route(route_type, _bridge_agent_request(data, agent, bridge_auth.litellm_params)) # Get API base URL from agent config - if not agent.agent_card_params or "url" not in agent.agent_card_params: + card_params: Final = as_str_mapping(agent.agent_card_params) + if not card_params or "url" not in card_params: verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False) # Inject API base and route to litellm - data["api_base"] = agent.agent_card_params["url"] + data["api_base"] = card_params["url"] verbose_proxy_logger.debug("[A2A] Routing %s to %s", model_name, data["api_base"]) return _call_route(route_type, data) diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 8e348423837..b0088d93112 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -21,6 +21,7 @@ import json import re from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Annotated, Final, Protocol, TypedDict, get_args from urllib.parse import urlsplit @@ -299,7 +300,7 @@ def _validate_installation_preference(spec: PluginSpec) -> None: def _build_plugin_manifest(name: str, spec: PluginSpec) -> Mapping[str, object]: """Build the stored manifest dict shared by plugin create and update.""" - dumped: Final[Mapping[str, object]] = spec.model_dump(exclude_none=True) + dumped: Final[Mapping[str, object]] = MappingProxyType(spec.model_dump(exclude_none=True)) return {"name": name, **{key: value for key, value in dumped.items() if value and key != "name"}} diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 73d20ae632c..6178839fc92 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -921,8 +921,9 @@ def get_request_route(request: Request) -> str: scope: Final = request.scope if not isinstance(scope, dict): return str(request.url.path) - raw_path: Final[str] = str(scope.get("path", request.url.path)) - root_path: Final[str] = str(scope.get("app_root_path", scope.get("root_path", ""))).rstrip("/") + scope_fields: Final[Mapping[str, object]] = MappingProxyType(scope) + raw_path: Final[str] = str(scope_fields.get("path", request.url.path)) + root_path: Final[str] = str(scope_fields.get("app_root_path", scope_fields.get("root_path", ""))).rstrip("/") if not isinstance(raw_path, str): return str(request.url.path) # Strip root_path only when it matches whole path segments — guarding @@ -1517,8 +1518,15 @@ def get_model_rate_limit_from_metadata( metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"], rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"], ) -> dict[str, int] | None: - if getattr(user_api_key_dict, metadata_accessor_key): - return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key) + metadata: Final = ( + user_api_key_dict.team_metadata + if metadata_accessor_key == "team_metadata" + else user_api_key_dict.organization_metadata + if metadata_accessor_key == "organization_metadata" + else user_api_key_dict.project_metadata + ) + if metadata: + return metadata.get(rate_limit_key) return None diff --git a/litellm/proxy/auth/ip_address_utils.py b/litellm/proxy/auth/ip_address_utils.py index f1ca2c70f3b..56efe113fc5 100644 --- a/litellm/proxy/auth/ip_address_utils.py +++ b/litellm/proxy/auth/ip_address_utils.py @@ -283,7 +283,7 @@ class IPAddressUtils: @staticmethod def get_mcp_client_ip( request: Request, - general_settings: dict[str, Any] | None = None, + general_settings: dict[str, object] | None = None, ) -> str | None: """ Extract client IP from a FastAPI request for MCP access control. @@ -371,4 +371,4 @@ class IPAddressUtils: return client_ip case _HopCountUnset(): pass - return get_request_ip_address(request, use_x_forwarded_for=use_xff) + return get_request_ip_address(request, use_x_forwarded_for=bool(use_xff)) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 720bfabb95a..3ae671d40ac 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -160,7 +160,7 @@ class RouteChecks: if allowed_route in LiteLLMRoutes._member_names_: if RouteChecks.check_route_access( route=route, - allowed_routes=LiteLLMRoutes._member_map_[allowed_route].value, + allowed_routes=LiteLLMRoutes[allowed_route].value, ): if ( allowed_route in _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 27fa4731baf..e1d98d8ddb6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -13,6 +13,7 @@ import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Final, NamedTuple, Protocol, Union, cast import fastapi @@ -2693,7 +2694,7 @@ async def user_api_key_auth_builder( and valid_token.last_refreshed_at is not None and team_obj.last_refreshed_at > valid_token.last_refreshed_at ): - team_obj_dict: Final = team_obj.__dict__ + team_obj_dict: Final[Mapping[str, object]] = MappingProxyType(team_obj.__dict__) for k, v in team_obj_dict.items(): field_name = f"team_{k}" diff --git a/litellm/proxy/client/chat.py b/litellm/proxy/client/chat.py index c027bf2fe98..ccdef44f9e8 100644 --- a/litellm/proxy/client/chat.py +++ b/litellm/proxy/client/chat.py @@ -59,7 +59,7 @@ class ChatClient: frequency_penalty: float | None = None, user: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a chat completion. @@ -184,7 +184,8 @@ class ChatClient: response.raise_for_status() # Parse SSE stream - for line in response.iter_lines(): + lines: Final[Iterator[object]] = response.iter_lines() + for line in lines: if line: line = _SSE_LINE.validate_python(line).decode("utf-8") if line.startswith("data: "): diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index ef73b4f0d2d..f93c0632358 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -4,9 +4,14 @@ from typing import Final import click import rich +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping from ...http_client import HTTPClient -from .auth import cli_context_obj +from ._cli_context import cli_context_values + +_DECODED_JSON: Final = TypeAdapter(object) @click.group() @@ -40,23 +45,25 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): lite encryption migrate --check # attestation scan, no writes lite encryption migrate # perform the migration """ - client: Final = HTTPClient(cli_context_obj(ctx)["base_url"], ctx.obj["api_key"]) + context: Final = cli_context_values(ctx) + client: Final = HTTPClient(context["base_url"], context["api_key"]) - if check_only: - response = client.request("GET", "/credentials/migrate-encryption/check") - else: - response = client.request( + response: Final = _DECODED_JSON.validate_python( + client.request("GET", "/credentials/migrate-encryption/check") + if check_only + else client.request( "POST", "/credentials/migrate-encryption", json={}, params={"dry_run": "true"} if dry_run else None, ) + ) rich.print_json(data=response) - report: Final = response.get("report", {}) if isinstance(response, dict) else {} + report: Final = as_str_mapping((as_str_mapping(response) or {}).get("report", {})) or {} residual: Final = report.get("residual_legacy") - if residual is not None and residual > 0: + if isinstance(residual, int | float) and residual > 0: rich.print(f"[yellow]Residual legacy values remaining: {residual}[/yellow]") elif residual == 0: rich.print("[green]No legacy values remaining (residual_legacy == 0).[/green]") diff --git a/litellm/proxy/client/credentials.py b/litellm/proxy/client/credentials.py index ebcd79457a6..649fe529936 100644 --- a/litellm/proxy/client/credentials.py +++ b/litellm/proxy/client/credentials.py @@ -75,7 +75,7 @@ class CredentialsManagementClient: credential_values: Mapping[str, object], return_request: bool = False, display_name: str | None = None, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a new credential. @@ -121,7 +121,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Delete a credential by name. @@ -182,7 +182,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get a credential by name. diff --git a/litellm/proxy/client/http_client.py b/litellm/proxy/client/http_client.py index aa0b986b1ad..96a170c75cb 100644 --- a/litellm/proxy/client/http_client.py +++ b/litellm/proxy/client/http_client.py @@ -4,6 +4,11 @@ from collections.abc import Mapping from typing import Any, Final import requests +from typing_extensions import TypedDict, Unpack + + +class _RequestOptions(TypedDict, total=False, extra_items=object): + pass class HTTPClient: @@ -29,7 +34,7 @@ class HTTPClient: data: Mapping[str, object] | list | bytes | None = None, json: Mapping[str, object] | list | None = None, headers: dict[str, str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_RequestOptions], ) -> Any: """Make an HTTP request to the LiteLLM proxy server. diff --git a/litellm/proxy/client/model_groups.py b/litellm/proxy/client/model_groups.py index fef307600c4..1555d170212 100644 --- a/litellm/proxy/client/model_groups.py +++ b/litellm/proxy/client/model_groups.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import requests @@ -31,7 +31,7 @@ class ModelGroupsManagementClient: headers["Authorization"] = f"Bearer {self._api_key}" return headers - def info(self, return_request: bool = False) -> list[dict[str, Any]] | requests.Request: + def info(self, return_request: bool = False) -> list[dict[str, object]] | requests.Request: """ Get detailed information about all model groups from the server. diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1b50f8c24f3..7342974c0cc 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -158,7 +158,7 @@ class ModelsManagementClient: model_id: str | None = None, model_name: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get information about a specific model by its ID or name. diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index 40cdb117476..b2640d2a765 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -1,11 +1,15 @@ import asyncio import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.common_utils.config_sync_pubsub import ( # noqa: F401 # legacy module exports ConfigSyncPubSub, _ConfigSyncPubSub, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -30,6 +34,7 @@ _pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong _in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES) _BACKOFF_INITIAL_SECONDS: Final = 5.0 _BACKOFF_MAX_SECONDS: Final = 60.0 +_DECODED_JSON: Final = TypeAdapter(object) def auth_cache_invalidation_channel(redis_cache: "RedisCache") -> str: @@ -46,7 +51,9 @@ class _CacheInvalidationMessage: def _cache_invalidation_message_json(cache_key: str, new_value: float | None = None, ttl: float | None = None) -> str: - message: Final = asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + message: Final[Mapping[str, object]] = MappingProxyType( + asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + ) return json.dumps({field: value for field, value in message.items() if value is not None}) @@ -62,10 +69,10 @@ def _message_from_data(data: object) -> _CacheInvalidationMessage | None: if not isinstance(data, str): return None try: - parsed: Final = json.loads(data) + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(data))) except json.JSONDecodeError: return None - if not isinstance(parsed, dict): + if parsed is None: return None cache_key: Final = parsed.get("cache_key") if not isinstance(cache_key, str): diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 46434a997b7..f889748c504 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from itertools import accumulate from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -39,6 +40,7 @@ from litellm.types.utils import ( ) _CALLBACK_VAR_MASKER: Final = SensitiveDataMasker() +_LOADED_CALLBACK: Final = TypeAdapter(object) # Compound names that are credential-bearing but don't contain any of the # default sensitive segments (so SensitiveDataMasker won't flag them). _EXTRA_SENSITIVE_CALLBACK_KEYS: Final = {"gcs_path_service_account"} @@ -389,9 +391,11 @@ def initialize_callbacks_on_proxy( imported_list.append( _loaded_callback_or_raise( entry=callback, - loaded=get_instance_fn( - value=callback, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=callback, + config_file_path=config_file_path, + ) ), ) ) @@ -408,9 +412,11 @@ def initialize_callbacks_on_proxy( litellm.callbacks = [ _loaded_callback_or_raise( entry=value, - loaded=get_instance_fn( - value=value, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=value, + config_file_path=config_file_path, + ) ), ) ] diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 521a9a15bd0..399d1231d7c 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -164,7 +164,7 @@ def coerce_numeric_form_fields( def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = orjson.loads(body) + parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 089d0b7a226..02da6f706d1 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -37,7 +37,7 @@ This module provides a single proxy-side error class that: import json from collections.abc import Mapping -from typing import Any, Final +from typing import Final from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError @@ -171,7 +171,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): def __init__( self, - detail: Any, + detail: object, headers: Mapping[str, object] | None = None, category: str | RateLimitErrorCategory = RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type: str | RateLimitType | None = None, diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index 35bd69ef0bd..a2ffdd4d59d 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -13,6 +13,7 @@ from typing import Final from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -26,6 +27,8 @@ from litellm.proxy.container_endpoints.ownership import ( get_container_forwarding_params, ) +_UPLOADED_FILE_FIELD: Final = TypeAdapter(object) + def _load_endpoints_config() -> dict: """Load the endpoints configuration from JSON file.""" @@ -304,7 +307,7 @@ async def _process_multipart_upload_request( raise HTTPException(status_code=400, detail="Missing required 'file' field") # convert_upload_files_to_file_data returns list of tuples, extract single file - file_list: Final = data["file"] + file_list: Final = _UPLOADED_FILE_FIELD.validate_python(data["file"]) if isinstance(file_list, list) and len(file_list) > 0: data["file"] = file_list[0] diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 75882bc3fb1..4ddcf7e600b 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -526,7 +526,7 @@ async def _upsert_turn_with_retry( except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise - await asyncio.sleep(2**attempt + random.uniform(0, 1)) + await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) else: return diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 627b30e83f7..1116cd604a3 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,7 +1,7 @@ import asyncio import json import logging -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeAlias from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -14,7 +14,9 @@ from litellm.types.services import ServiceTypes POD_LOCK_TARGET: Final = "pod_lock" if TYPE_CHECKING: - ProxyLogging = Any + from litellm.proxy.utils import ProxyLogging as _ProxyLogging + + ProxyLogging: TypeAlias = _ProxyLogging else: ProxyLogging = Any diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index 48ad57fd06e..52a0be3d49e 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -298,7 +298,7 @@ class RoutingPrismaWrapper: def __getattr__(self, name: str) -> Any: if name in _TOP_LEVEL_READ_METHODS: - return getattr(self.read_target, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self.read_target, name)) writer_attr: Final = _DELEGATED_ATTRIBUTE.validate_python(getattr(self._writer, name)) # Per-model action accessors are non-callable instances that expose # both `find_many` and `create`. Methods like execute_raw / batch_ / diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 70e7aac12b5..a5822bfb0e3 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -90,6 +90,7 @@ if TYPE_CHECKING: router: Final = APIRouter() GUARDRAIL_REGISTRY: Final = GuardrailRegistry() _DECODED_JSON: Final = TypeAdapter(object) +_DECODED_YAML: Final = TypeAdapter(object) def _as_str_object_mapping(mapping: Mapping[str, object]) -> Mapping[str, object]: @@ -1682,7 +1683,7 @@ async def validate_blocked_words_file(request: dict[str, str]): if not file_content: return {"valid": False, "error": "No file content provided"} - data: Final = yaml.safe_load(file_content) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(file_content)) if not isinstance(data, dict) or "blocked_words" not in data: return { diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py index ac0a6911184..aabac874ca2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py @@ -63,6 +63,7 @@ _INVALID_ASSERTION_AADSTS_PREFIX: Final = "50027" _AADSTS_CODES_ADAPTER: Final = TypeAdapter(tuple[int, ...]) _MCP_CALL_TYPES: Final[tuple[str, ...]] = ("mcp_call", "call_mcp_tool") _TOOL_INPUT_SCHEMA_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) _OBO_CACHE_MAX_ENTRIES: Final = 1000 _DEFAULT_TOKEN_TTL_SECONDS: Final = 3599.0 _TOKEN_EXPIRY_SLACK_SECONDS: Final = 60.0 @@ -343,7 +344,7 @@ class Agent365Guardrail(CustomGuardrail): latency_ms: float, ) -> dict: # mutable-ok: returns the request data dict per hook contract try: - parsed_verdict: Final = response.json() + parsed_verdict: Final = _DECODED_JSON.validate_python(response.json()) except ValueError: return self._handle_unavailable( data=data, @@ -486,7 +487,7 @@ class Agent365Guardrail(CustomGuardrail): response=response, ) try: - parsed_body: Final = response.json() + parsed_body: Final = _DECODED_JSON.validate_python(response.json()) except ValueError as exc: raise Agent365MalformedResponseError("the Entra token endpoint returned a non-JSON body") from exc if not isinstance(parsed_body, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py index 993c1d30949..389706d7417 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py +++ b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py @@ -2,12 +2,13 @@ import os import uuid from collections.abc import Mapping from typing import ( - Any, # noqa: TID251 # the only type CustomGuardrail.__init__ accepts for its open-ended kwargs Final, Literal, ) import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -36,6 +37,12 @@ DEFAULT_BLOCKED_MESSAGE: Final = "Blocked by your organization's content policy. SUPPORTED_EVENT_HOOKS: Final = (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call) +_DECODED_JSON: Final = TypeAdapter(object) + + +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + def _stream_cannot_carry_a_rewrite(request_data: Mapping[str, object] | None) -> bool: """Whether this is a streamed response on a route the framework can only block, not rewrite. @@ -97,7 +104,7 @@ class AiriaGuardrail(CustomGuardrail): timeout: float | None = None, supported_event_hooks: list[GuardrailEventHooks] | None = None, # mutable-ok: matches CustomGuardrail.__init__ async_handler: AsyncHTTPHandler | None = None, # dependency-injected in tests; defaults to a real client - **kwargs: Any, # kwargs-ok: passed straight through to CustomGuardrail.__init__ + **kwargs: Unpack[_CustomGuardrailOptions], # kwargs-ok: passed straight through to CustomGuardrail.__init__ ) -> None: resolved_timeout: Final = timeout or float(os.getenv("AIRIA_TIMEOUT", "10")) @@ -161,7 +168,7 @@ class AiriaGuardrail(CustomGuardrail): headers={"Authorization": f"Bearer {self.api_key}"}, ) response.raise_for_status() - raw_body: Final[object] = response.json() + raw_body: Final = _DECODED_JSON.validate_python(response.json()) body: Final = _as_object(raw_body) if body is None: raise TypeError(f"expected a JSON object in the response body, got {type(raw_body).__name__}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 029feba89fb..4d910cde603 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -28,6 +28,7 @@ from urllib.parse import urlparse import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -72,6 +73,7 @@ if TYPE_CHECKING: ) BYPASS_HEADER: Final = "x-compresr-bypass" +_AGENTIC_LOOP_MODEL: Final = TypeAdapter(object) COMPRESR_RETRIEVE_TOOL_NAME: Final = "compresr_retrieve" DEFAULT_API_BASE: Final = "https://api.compresr.ai" DEFAULT_COMPRESSION_MODEL: Final = "latte_v2" @@ -1118,7 +1120,7 @@ class CompresrGuardrail(CustomGuardrail): full_model_name = model if logging_obj is not None: agentic_params: Final = getattr(logging_obj, "model_call_details", {}).get("agentic_loop_params", {}) - candidate: Final = agentic_params.get("model", model) + candidate: Final = _AGENTIC_LOOP_MODEL.validate_python(agentic_params.get("model", model)) if isinstance(candidate, str) and candidate: full_model_name = candidate diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 199da7df30d..ecd23d86540 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -21,6 +21,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, @@ -126,6 +127,13 @@ def _sanitize_inbound_headers( return sanitized or None +def _logged_inbound_headers(details: Mapping[str, object]) -> Mapping[object, object] | None: + litellm_params: Final = as_str_mapping(details.get("litellm_params", {})) + metadata: Final = as_str_mapping(litellm_params.get("metadata", {})) if litellm_params is not None else None + headers: Final = metadata.get("headers", None) if metadata is not None else None + return headers if isinstance(headers, dict) else None + + def _extract_inbound_headers( request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"], @@ -163,7 +171,7 @@ def _extract_inbound_headers( if logging_obj and getattr(logging_obj, "model_call_details", None): try: details: Final = logging_obj.model_call_details or {} - headers = details.get("litellm_params", {}).get("metadata", {}).get("headers", None) + headers = _logged_inbound_headers(details) if headers: return _sanitize_inbound_headers(headers, extra_allowlist=extra_allowlist) except Exception: @@ -283,7 +291,7 @@ class GenericGuardrailAPI(CustomGuardrail): ) self.num_retries: int = num_retries or 0 self.headers = headers or {} - self.extra_headers = extra_headers or [] + self.extra_headers: Sequence[object] = extra_headers or [] # If api_key is provided, add it as x-api-key header if api_key: diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index f32228a6204..7ff5aa2466b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -34,6 +34,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj GRAYSWAN_BLOCK_ERROR_MSG: Final = "Blocked by Gray Swan Guardrail" +_DECODED_JSON: Final = TypeAdapter(object) class _GraySwanMonitorResponse(TypedDict): @@ -705,7 +706,9 @@ class GraySwanGuardrail(CustomGuardrail): {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers ) if cleaned_litellm_metadata: - sanitized: Final[object] = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + sanitized: Final = _DECODED_JSON.validate_python( + safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + ) if isinstance(sanitized, dict) and sanitized: payload["litellm_metadata"] = sanitized diff --git a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py index d5edcc19a02..a1d0900ebed 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py +++ b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Final from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -23,6 +24,8 @@ from litellm.types.utils import CallTypesLiteral, GuardrailStatus if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class JavelinGuardrail(CustomGuardrail): @classmethod @@ -113,7 +116,9 @@ class JavelinGuardrail(CustomGuardrail): json=dict(request), timeout=self.timeout, ) - verbose_proxy_logger.debug("Javelin Guardrail: Javelin guard API response: %s", response.json()) + verbose_proxy_logger.debug( + "Javelin Guardrail: Javelin guard API response: %s", _DECODED_JSON.validate_python(response.json()) + ) response_data: Final = response.json() # Ensure the response has the required assessments field if "assessments" not in response_data: diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py index 8efd1dc79e3..557c8ca496f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -40,6 +41,8 @@ from litellm.types.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import ( ) from litellm.types.utils import CallTypesLiteral, GuardrailStatus, ModelResponse +_DECODED_JSON: Final = TypeAdapter(object) + _DETECTOR_CATEGORY_PHRASES: Final[Mapping[str, str]] = MappingProxyType( { "prompt_injection": "a potential prompt injection attempt", @@ -402,7 +405,9 @@ class LakeraAIGuardrail(CustomGuardrail): json=request, timeout=self.timeout, ) - verbose_proxy_logger.debug("Lakera AI v2 guard response: %s", response.json()) + verbose_proxy_logger.debug( + "Lakera AI v2 guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) lakera_response = LakeraAIResponse(**response.json()) return lakera_response, masked_entity_count except Exception as e: diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index e05db5003fa..97b35769f63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -9,7 +9,7 @@ import json import os import uuid from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict +from typing import TYPE_CHECKING, Final, Literal, TypedDict try: import ulid @@ -39,6 +39,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.integrations.custom_guardrail import dc as global_cache +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -397,13 +398,15 @@ class LassoGuardrail(CustomGuardrail): HTTPException: If blocking violations are detected """ raw_messages: Final[list[dict[str, object]]] = data.get("messages") or [] - messages: list[dict[str, Any]] = self._expand_messages_for_classification(raw_messages) if raw_messages else [] - messages_count: Final = len(messages) - if data.get("input") is not None: - # Responses-API payloads carry text in data["input"]. Inspect it - # alongside any "messages" array — otherwise a caller can attach - # benign messages and stash blocked content in input to bypass. - messages.extend(build_inspection_messages({"input": data["input"]})) + expanded_messages: Final = self._expand_messages_for_classification(raw_messages) if raw_messages else [] + messages_count: Final = len(expanded_messages) + # Responses-API payloads carry text in data["input"]. Inspect it + # alongside any "messages" array — otherwise a caller can attach + # benign messages and stash blocked content in input to bypass. + input_messages: Final = ( + build_inspection_messages({"input": data["input"]}) if data.get("input") is not None else [] + ) + messages: Final[Sequence[Mapping[str, object]]] = [*expanded_messages, *input_messages] if not messages: return data @@ -421,7 +424,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], ) -> dict: """Handle classification without masking.""" try: @@ -439,7 +442,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], messages_count: int, ) -> dict: """Handle masking with classifix endpoint. @@ -470,7 +473,7 @@ class LassoGuardrail(CustomGuardrail): text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)] if text_parts: data["input"] = "\n".join(text_parts) - self._log_masking_applied(message_type, dict(response)) + self._log_masking_applied(message_type, response) return data except Exception as e: @@ -479,7 +482,7 @@ class LassoGuardrail(CustomGuardrail): def _map_masked_messages_back( self, - original_messages: list[dict[str, Any]], + original_messages: Sequence[Mapping[str, object]], masked_messages: Sequence[Mapping[str, object]], ) -> list[dict[str, object]]: """Map Lasso-format masked messages back onto the original OpenAI-format messages. @@ -492,7 +495,7 @@ class LassoGuardrail(CustomGuardrail): """ # Index masked content by type so we can look up by id without caring about order. masked_tool_use: Final[dict[object, dict[str, object]]] = {} - masked_tool_result: Final[dict[str, str]] = {} + masked_tool_result: Final[dict[object, str]] = {} masked_text: Final[list[str]] = [] for msg in masked_messages: @@ -567,12 +570,12 @@ class LassoGuardrail(CustomGuardrail): def _update_tool_calls_from_masked( self, - tool_calls: list[object], + tool_calls: object, masked_tool_use: Mapping[object, Mapping[str, object]], ) -> list[object]: """Replace tool_call arguments with masked values returned by Lasso.""" updated: Final = [] - for call in tool_calls: + for call in iter_items(tool_calls): call_id = self._get_field(call, "id") if call_id and call_id in masked_tool_use: masked_input = masked_tool_use[call_id].get("input") @@ -627,7 +630,7 @@ class LassoGuardrail(CustomGuardrail): def _log_masking_applied( self, message_type: Literal["PROMPT", "COMPLETION"], - response: dict[str, Any], + response: LassoResponse, ) -> None: """Log masking application with structured context.""" conversation_id: Final = getattr(self, "conversation_id", "unknown") @@ -637,12 +640,12 @@ class LassoGuardrail(CustomGuardrail): "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, "violations_count": len(response.get("findings", {})), - "masked_fields": len(response.get("messages", [])), + "masked_fields": len(response.get("messages") or []), "conversation_id": conversation_id, }, ) - def _expand_messages_for_classification(self, messages: list[dict[str, Any]]) -> list[dict[str, object]]: + def _expand_messages_for_classification(self, messages: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: """ Convert raw OpenAI-format messages to Lasso API format with content blocks. @@ -701,7 +704,7 @@ class LassoGuardrail(CustomGuardrail): expanded.append({"role": role, "content": content}) if role == "assistant": - for call in msg.get("tool_calls") or []: + for call in iter_items(msg.get("tool_calls") or []): call_id, name, input_data = self._extract_tool_call_fields(call) if not call_id or not name: verbose_proxy_logger.warning( @@ -749,7 +752,7 @@ class LassoGuardrail(CustomGuardrail): def _prepare_payload( self, - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"] = "PROMPT", diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py index 5e2b85250d2..76c3a86d701 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py @@ -13,6 +13,9 @@ import json from pathlib import Path from typing import Any, Final +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base import ( # noqa: F401 # legacy module exports BaseCompetitorIntentChecker, _compile_marker, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -114,6 +117,7 @@ AIRLINE_EXPLICIT_OTHER_MEANING_MARKER: Final = ( ) _MAJOR_AIRLINES_PATH: Final = Path(__file__).resolve().parent / "major_airlines.json" +_DECODED_JSON: Final = TypeAdapter(object) def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: @@ -127,11 +131,11 @@ def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: return [] try: with open(_MAJOR_AIRLINES_PATH, encoding="utf-8") as f: - airlines: Final = json.load(f) + airlines: Final = _DECODED_JSON.validate_python(json.load(f)) except (json.JSONDecodeError, OSError): return [] result: Final[list[str]] = [] - for entry in airlines: + for entry in iter_items(airlines): if not isinstance(entry, dict): continue match_str = entry.get("match") or "" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 40e44151e65..32d43af79a8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -806,7 +806,7 @@ class ContentFilterGuardrail(CustomGuardrail): """ try: with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_ARGUMENTS.validate_python(yaml.safe_load(f)) if not isinstance(data, dict) or "blocked_words" not in data: raise ValueError("Invalid format: file must contain 'blocked_words' key with list of words") diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py index 4ef8bbdbb3e..ec713f76211 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py @@ -10,6 +10,8 @@ from typing import ( TypeAlias, ) +from pydantic import TypeAdapter + from .payload import ( JsonBody, MutableRequest, @@ -24,6 +26,8 @@ from .payload import ( write_field, ) +_OPAQUE_VALUE: Final = TypeAdapter(object) + ANTHROPIC_DELTA_FIELDS: Final = MappingProxyType({"text_delta": "text", "input_json_delta": "partial_json"}) SSE_EVENT_BOUNDARY: Final = re.compile(rb"(\r?\n\r?\n)") @@ -173,7 +177,7 @@ class AnthropicSSERestorer: return block line: Final = lines[data_lines[0]] try: - parsed: Final[object] = json.loads(line[len("data:") :]) + parsed: Final = _OPAQUE_VALUE.validate_python(json.loads(line[len("data:") :])) except ValueError: return block event: Final = as_object(parsed) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py index 221c4b3752b..2e3f7120726 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py @@ -74,14 +74,14 @@ import os import re import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Final, Optional import httpx import jwt from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -90,7 +90,6 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypesLiteral @@ -98,6 +97,10 @@ if TYPE_CHECKING: from jwt.types import Options +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _OIDCDiscoveryDocument(TypedDict, total=False): jwks_uri: str @@ -279,11 +282,13 @@ class MCPJWTSigner(CustomGuardrail): debug_headers: bool = False, # FR-10: Configurable scopes allowed_scopes: list[str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) # --- Signing key setup --- key_material: Final = os.environ.get(self.SIGNING_KEY_ENV) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 8482567b6da..3cb20a00e2f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -6,7 +6,7 @@ on the LiteLLM gateway. Blocks or alerts when unregistered servers are found. """ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal from fastapi import HTTPException @@ -21,6 +21,9 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( ) from litellm.types.guardrails import GuardrailEventHooks +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + class MCPSecurityGuardrail(CustomGuardrail): def __init__( @@ -40,7 +43,7 @@ class MCPSecurityGuardrail(CustomGuardrail): async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, - cache: Any, + cache: "DualCache", data: dict, call_type: str, ) -> Exception | str | dict | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py index ed955ac829d..06de4ef0e93 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Final +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.guardrails import SupportedGuardrailIntegrations from .purview_dlp import MicrosoftPurviewDLPGuardrail @@ -11,8 +12,8 @@ if TYPE_CHECKING: def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm - tenant_id: Final = getattr(litellm_params, "tenant_id", None) - client_id: Final = getattr(litellm_params, "client_id", None) + tenant_id: Final = attribute_or_none(litellm_params, "tenant_id") + client_id: Final = attribute_or_none(litellm_params, "client_id") # client_secret can be passed via the standard api_key field or as # a dedicated client_secret parameter. diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py index 718401b6e68..43eb54d1464 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py @@ -10,9 +10,12 @@ import json import os from collections.abc import Callable, Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional, TypeAlias, cast from urllib.parse import urlparse +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -25,7 +28,6 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.noma import NomaV2StreamingParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus @@ -39,6 +41,7 @@ _DEFAULT_API_BASE: Final = "https://api.noma.security/" _AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail" _INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls") _DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname +_DECODED_JSON: Final = TypeAdapter(object) _GuardrailJsonResponse: TypeAlias = Exception | str | dict[str, object] @@ -50,6 +53,10 @@ _LOGGING_KEYS_DUPLICATING_SCAN_INPUTS: Final = _KEYS_DUPLICATING_SCAN_INPUTS + ( ) +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _Action(str, enum.Enum): BLOCKED = "BLOCKED" NONE = "NONE" @@ -74,7 +81,7 @@ class NomaV2Guardrail(CustomGuardrail): streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, streaming_buffer_release_on_scan: bool | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) @@ -115,10 +122,11 @@ class NomaV2Guardrail(CustomGuardrail): if streaming.streaming_buffer_release_on_scan is not None: self.streaming_buffer_release_on_scan: bool = streaming.streaming_buffer_release_on_scan - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -219,7 +227,7 @@ class NomaV2Guardrail(CustomGuardrail): except (ValueError, TypeError): json_str = safe_dumps(payload) - safe_payload: Final[object] = safe_json_loads(json_str, default={}) + safe_payload: Final = _DECODED_JSON.validate_python(safe_json_loads(json_str, default={})) if safe_payload == {} and payload: verbose_proxy_logger.warning( "Noma v2 guardrail: payload serialization failed, falling back to empty payload" diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 1a6feb47215..7c26c4c4c56 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Final, Literal, Optional import httpx from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -27,6 +28,8 @@ from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class OnyxGuardrail(CustomGuardrail): @classmethod @@ -115,7 +118,7 @@ class OnyxGuardrail(CustomGuardrail): try: response: Final = ModelResponse(**request_data) parsed: Final = response.json() - payload = parsed.get("response", {}) + payload = _DECODED_JSON.validate_python(parsed.get("response", {})) except Exception as e: verbose_proxy_logger.error( "Error in converting request_data to ModelResponse: %s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 7af5eaf1da9..98be35079c3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -40,6 +41,8 @@ if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIModerationResponse from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + def _config_secret(value: object) -> object: return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value @@ -151,7 +154,9 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): timeout=self.timeout, ) - verbose_proxy_logger.debug("OpenAI Moderation guard response: %s", response.json()) + verbose_proxy_logger.debug( + "OpenAI Moderation guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) if response.status_code != 200: raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 8e03f7db4f5..4fe14937c75 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -894,7 +894,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not line.startswith("data: "): continue try: - data = json.loads(line[6:]) + data = _DECODED_ARGUMENTS.validate_python(json.loads(line[6:])) except (json.JSONDecodeError, ValueError): continue if not isinstance(data, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index cadc9ca689b..7e2f6bfec33 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -85,6 +85,7 @@ class _PresidioAnonymizeResponse(TypedDict): _PRESIDIO_ANONYMIZE_ADAPTER: Final[TypeAdapter[_PresidioAnonymizeResponse | None]] = TypeAdapter( _PresidioAnonymizeResponse | None ) +_DECODED_JSON: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -1595,10 +1596,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if line.startswith("data: ") and line != "data: [DONE]": raw_json = line[6:] try: - event = json.loads(raw_json) + event = _DECODED_JSON.validate_python(json.loads(raw_json)) delta = event.get("delta") if isinstance(event, dict) else None if ( - isinstance(delta, dict) + isinstance(event, dict) + and isinstance(delta, dict) and event.get("type") == "content_block_delta" and delta.get("type") == "text_delta" and isinstance(delta.get("text"), str) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a92e67046bd..6150fccac0c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -10,6 +10,7 @@ import os from typing import Any, Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -29,6 +30,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs GUARDRAIL_NAME: Final = "qualifire" DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai" +_DECODED_JSON: Final = TypeAdapter(object) class QualifireGuardrail(CustomGuardrail): @@ -188,7 +190,7 @@ class QualifireGuardrail(CustomGuardrail): args = function_info.get("arguments", {}) if isinstance(args, str): try: - args = json.loads(args) + args = _DECODED_JSON.validate_python(json.loads(args)) except json.JSONDecodeError: args = {} api_tool_calls.append( diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index 1a571047977..342e5b64714 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import yaml +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD @@ -22,6 +23,7 @@ if TYPE_CHECKING: ROUTE_TEMPLATES_DIR: Final = os.path.join(os.path.dirname(__file__), "route_templates") +_DECODED_YAML: Final = TypeAdapter(object) class SemanticGuardRouteLoader: @@ -55,7 +57,7 @@ class SemanticGuardRouteLoader: if not os.path.exists(file_path): raise ValueError(f"SemanticGuard: custom routes file not found: {file_path}") with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(f)) if isinstance(data, list): return data if isinstance(data, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 3ad9d1a2c42..9e46af1c1ca 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, runtime import anyio from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -50,6 +51,8 @@ A2A_CALL_TYPES: Final = (CallTypes.asend_message, CallTypes.send_message) GUARDRAIL_NAME: Final = "unified_llm_guardrails" +_DECODED_JSON: Final = TypeAdapter(object) + _RequestData: TypeAlias = dict[str, object] @@ -183,7 +186,7 @@ def _get_a2a_request_id(responses_so_far: Sequence[object], request_data: dict) return item.get("id") if isinstance(item, str): try: - obj = json.loads(item.strip()) + obj = _DECODED_JSON.validate_python(json.loads(item.strip())) if isinstance(obj, dict) and "id" in obj: return obj.get("id") except (json.JSONDecodeError, TypeError): diff --git a/litellm/proxy/guardrails/tool_name_extraction.py b/litellm/proxy/guardrails/tool_name_extraction.py index 526f58dcd36..c49835905fb 100644 --- a/litellm/proxy/guardrails/tool_name_extraction.py +++ b/litellm/proxy/guardrails/tool_name_extraction.py @@ -6,8 +6,8 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl with standalone extractors for generate_content and MCP. """ -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Callable, Mapping +from typing import Final from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings @@ -15,7 +15,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import list_field from litellm.types.utils import CallTypes # Call types that have no guardrail translation handler; we use standalone extractors -STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {} +STANDALONE_EXTRACTORS: Final[dict[str, Callable[[Mapping[str, object]], list[str]]]] = {} def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]: @@ -28,7 +28,7 @@ def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str return names -def _extract_mcp_tool_names(data: dict) -> list[str]: +def _extract_mcp_tool_names(data: Mapping[str, object]) -> list[str]: """MCP call_tool: name or mcp_tool_name in body""" names: Final[list[str]] = [] name: Final = data.get("name") or data.get("mcp_tool_name") diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index ef60a800cdd..fba822d1c1c 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -449,10 +449,10 @@ class KeyManagementEventHooks: try: from litellm.proxy import proxy_server as proxy_server_module + from litellm.proxy.proxy_server import prisma_client except ImportError: return None - prisma_client: Final = getattr(proxy_server_module, "prisma_client", None) user_api_key_cache: Final = getattr(proxy_server_module, "user_api_key_cache", None) if prisma_client is None or user_api_key_cache is None: diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 47ea50debc9..05c64317a44 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -892,7 +892,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): {**_hidden_params, "additional_headers": _additional_headers}, ) - return await super().async_post_call_success_hook(data, user_api_key_dict, response) + await super().async_post_call_success_hook(data, user_api_key_dict, response) PROXY_MaxParallelRequestsHandler: Final = _PROXY_MaxParallelRequestsHandler diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index d858eff3348..6710501bc5b 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -97,7 +97,7 @@ if TYPE_CHECKING: from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fc0868a5fc9..11f616a5e18 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3217,12 +3217,13 @@ async def update_useful_links( ) -def _validated_labeled_tiers( - tier_labels: dict[ComplexityTier, str], # mutable-ok: Pydantic materializes JSON object fields as dicts -) -> tuple[tuple[ComplexityTier, str], ...]: +_DECODED_JSON: Final = TypeAdapter(object) + + +def _validated_labeled_tiers(tier_labels: object) -> tuple[tuple[ComplexityTier, str], ...]: """Validate tier labels once for both prompt-preview transports.""" try: - return ComplexityRouterConfig(tier_labels=tier_labels).labeled_tiers() + return ComplexityRouterConfig.model_validate({"tier_labels": tier_labels}).labeled_tiers() except (TypeError, ValidationError) as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", @@ -3237,7 +3238,7 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity if not tier_labels: return None try: - parsed: Final = json.loads(tier_labels) + parsed: Final = _DECODED_JSON.validate_python(json.loads(tier_labels)) except JSONDecodeError as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0595d23bd9a..a6c543fde8f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3050,7 +3050,7 @@ async def _process_team_members( # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models member_allowed_models = data.allowed_models - team_default_member_models: Final = getattr(complete_team_data, "default_team_member_models", None) + team_default_member_models: Final = complete_team_data.default_team_member_models if member_allowed_models is None and team_default_member_models: member_allowed_models = team_default_member_models diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 4cc849762aa..788baff784c 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1605,7 +1605,7 @@ async def get_generic_sso_response( team_mappings: Final = await _setup_team_mappings() generic_include_token_claims: Final = os.getenv("GENERIC_INCLUDE_TOKEN_CLAIMS", "false").lower() == "true" - def response_convertor(response: Mapping[str, object], httpx_session: object): + def response_convertor(response: Mapping[str, object], httpx_session: object) -> CustomOpenID: nonlocal received_response # return for user debugging response_id_token: Final = response.get("id_token") response_access_token: Final = response.get("access_token") diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 3468396c6f6..920970427ba 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -310,7 +310,7 @@ async def _resolve_mcp_server_identifiers_to_ids( identifiers=identifiers, prisma_client=prisma_client, ): - server_id = getattr(server, "server_id", None) + server_id = server.server_id if not server_id: continue for identifier in identifiers: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index d9d3b6f8c92..3b9e8d4c227 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1305,7 +1305,7 @@ async def handle_bedrock_count_tokens( verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) from litellm.litellm_core_utils.llm_response_utils.get_headers import get_response_headers - provider_headers: Final = getattr(getattr(e, "response", None), "headers", None) + provider_headers: Final = e.response.headers raise HTTPException( status_code=e.status_code, detail={"error": e.message}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py index e277655f1e1..2afc1d17bc2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -36,6 +36,7 @@ from litellm.types.passthrough_endpoints.tinyfish import ( from litellm.types.utils import StandardPassThroughResponseObject _RUN_ADAPTER: Final = TypeAdapter(TinyfishRun) +_DECODED_JSON: Final = TypeAdapter(object) _EMPTY_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) @@ -323,7 +324,7 @@ class TinyFishPassthroughLoggingHandler: "TinyFish passthrough: GET /v1/runs/%s returned %s", safe_run_id, response.status_code ) return None - payload: Final[object] = response.json() # any-ok: httpx Response.json() -> Any + payload: Final = _DECODED_JSON.validate_python(response.json()) return _parse_run(payload) except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request verbose_proxy_logger.warning("[Non blocking logging error] TinyFish run fetch failed: %s", e) @@ -413,7 +414,7 @@ def _run_id_from_sse_chunks(all_chunks: Sequence[str]) -> str | None: if not line.startswith("data:"): continue try: - event_payload: object = json.loads(line[5:].strip()) # any-ok: json.loads -> Any + event_payload: object = _DECODED_JSON.validate_python(json.loads(line[5:].strip())) except json.JSONDecodeError: continue event = _parse_run(event_payload) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py index 79f6dee90f7..5b45316a44c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py @@ -492,6 +492,10 @@ def s3_media_url(media_uri: str, aws_region_name: str) -> str | None: return f"https://{bucket}.s3.{aws_region_name}.{dns_suffix}/{quote(key)}" +def _header_or_default(headers: Mapping[str, str], name: str, default: str) -> str: + return headers.get(name, default) + + def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> bool: try: modified_at: Final = parsedate_to_datetime(headers["last-modified"]).timestamp() @@ -501,7 +505,7 @@ def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> boo async def write_media_within_limit(response: httpx.Response, media_file: IO[bytes], max_bytes: int) -> bool: - if int(response.headers.get("content-length", "0")) > max_bytes: + if int(_header_or_default(response.headers, "content-length", "0")) > max_bytes: return False async for chunk in response.aiter_bytes(): _ = media_file.write(chunk) @@ -591,8 +595,7 @@ class TranscribePassthroughLoggingHandler: @staticmethod def _operation_from_response(httpx_response: httpx.Response) -> str: - headers: Final[Mapping[str, str]] = httpx_response.request.headers - target: Final[str] = headers.get("x-amz-target", "") + target: Final = _header_or_default(httpx_response.request.headers, "x-amz-target", "") return target.split(".")[-1] @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 59b8d40577a..e70d583a027 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -7,6 +7,7 @@ records the reported values without recomputing them. """ import math +from collections.abc import Mapping from dataclasses import dataclass from typing import Final @@ -89,7 +90,7 @@ def _parse_total_tokens(raw_value: str | None) -> int | None: return total_tokens -def parse_upstream_reported_usage(headers: httpx.Headers) -> UpstreamReportedUsage | None: +def parse_upstream_reported_usage(headers: Mapping[str, str]) -> UpstreamReportedUsage | None: """Read the reported totals off an upstream pass-through response. Returns ``None`` when neither header is present, which is the normal case diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5128ddab87..c1a0e633fbb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1140,7 +1140,7 @@ def _redact_worker_config_for_logging(worker_config: str | dict[str, JsonValue] return None if isinstance(worker_config, dict): return _redact_secret_values_in_obj(worker_config) - parsed: Final = safe_json_loads(worker_config, default=None) + parsed: Final[JsonValue] = safe_json_loads(worker_config, default=None) if isinstance(parsed, dict): return safe_dumps(_redact_secret_values_in_obj(parsed)) return worker_config @@ -8517,8 +8517,12 @@ class ProxyConfig: environment_read: Final = next(self._environment_read_numbers) sections: Final = tuple(self._settings_stores) responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections)) + response: _ConfigParamValueRow | None for section, response in zip(sections, responses): - if response is None or (param_value := getattr(response, "param_value", None)) is None: + if response is None: + continue + param_value: JsonValue = getattr(response, "param_value", None) + if param_value is None: continue verbose_proxy_logger.debug( @@ -8545,7 +8549,10 @@ class ProxyConfig: self, prisma_client: PrismaClient, row: object, environment_before_db: Mapping[str, str], read: int ) -> None: current: Final = await self._environment_row_not_behind_the_applied_one(prisma_client, row) - if current is None or (param_value := getattr(current, "param_value", None)) is None: + if current is None: + return + param_value: Final[object] = getattr(current, "param_value", None) + if param_value is None: return self._apply_db_environment_variables(param_value, _config_row_revision(current), environment_before_db, read) diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index ea543aeae6e..4abc79febe1 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -21,6 +21,8 @@ from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import ( @@ -135,6 +137,9 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: return str(getattr(row, "model_name", "") or "") +_DECODED_JSON: Final = TypeAdapter(object) + + def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """A deployment's model_info as a mapping, decoding a JSON string, else None. @@ -143,7 +148,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """ if isinstance(raw, str): try: - decoded: Final[object] = json.loads(raw) + decoded: Final = _DECODED_JSON.validate_python(json.loads(raw)) except (TypeError, ValueError): return None return decoded if isinstance(decoded, dict) else None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a8bd35644c9..565b6c4cce8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1246,7 +1246,8 @@ def _timed_out_reading_from_provider(exception: Exception) -> bool: def _client_disconnected(exception: Exception) -> bool: """The 499 the proxy raises when the caller hung up while the provider was still working on the request.""" - return getattr(exception, "status_code", None) == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + status_code: Final[object] = getattr(exception, "status_code", None) + return status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED def _provider_billed_the_failure(exception: Exception) -> bool: diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index d4643994043..6d9249d5572 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from litellm.proxy._types import * @@ -36,6 +37,7 @@ from litellm.types.videos.utils import ( ) router: Final = APIRouter() +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) @router.post( @@ -582,7 +584,9 @@ async def video_create_character( if video_file: data["video"] = video_file[0] - target_model_name: Final = extract_model_from_target_model_names(data.get("target_model_names")) + target_model_name: Final = extract_model_from_target_model_names( + _OPAQUE_VALUE_ADAPTER.validate_python(data.get("target_model_names")) + ) if target_model_name and not data.get("model"): data["model"] = target_model_name diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index a2be42f1b76..56123e4bee3 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from litellm.router import Router -def extract_model_from_target_model_names(target_model_names: Any) -> str | None: +def extract_model_from_target_model_names(target_model_names: object) -> str | None: if isinstance(target_model_names, str): model_names: Final = [m.strip() for m in target_model_names.split(",") if m.strip()] return model_names[0] if model_names else None diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py index 1e7aa6ac1c6..5520912dd41 100644 --- a/litellm/rag/ingestion/bedrock_ingestion.py +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -106,11 +106,11 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ) self.embedding_model = self.vector_store_config.get("embedding_model") or "amazon.titan-embed-text-v2:0" - self.wait_for_ingestion = self.vector_store_config.get("wait_for_ingestion", False) + self.wait_for_ingestion: object = self.vector_store_config.get("wait_for_ingestion", False) self.ingestion_timeout: int = _get_int(self.vector_store_config.get("ingestion_timeout"), 300) # Get AWS region using BaseAWSLLM method - _aws_region: Final = self.vector_store_config.get("aws_region_name") + _aws_region: Final[object] = self.vector_store_config.get("aws_region_name") self.aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( aws_region_name=str(_aws_region) if _aws_region else None ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 5c9e5cac5ab..1e793f8995d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -5,6 +5,7 @@ from itertools import filterfalse from typing import Any, Final, cast import litellm +from litellm.litellm_core_utils.core_helpers import attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.custom_tools import ( @@ -1361,7 +1362,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: return False - return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ()) + return not any(getattr(part, "text", None) for part in attribute_items(item, "content")) def _emit_terminal_response_event( self, litellm_model_response: ModelResponse diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index bcd917c69d5..ec6b91290bb 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -28,7 +28,7 @@ from typing import ( import httpx from openai._streaming import SSEDecoder -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs import litellm @@ -163,6 +163,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams: _SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache" _UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text" +_DECODED_JSON: Final = TypeAdapter(object) def _load_json_object(payload: str | bytes) -> dict[str, object]: @@ -172,7 +173,7 @@ def _load_json_object(payload: str | bytes) -> dict[str, object]: def _load_json_value(payload: str | bytes) -> object: """Parse a JSON payload whose top-level shape the caller narrows itself.""" - return json.loads(payload) + return _DECODED_JSON.validate_python(json.loads(payload)) def _model_id_from_metadata(litellm_metadata: Mapping[str, object] | None) -> str | None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 503d85aaed9..c3f2f837cb1 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -5,10 +5,12 @@ from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel, TypeAdapter +from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( @@ -31,6 +33,10 @@ from litellm.types.utils import ( ) +class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): + pass + + def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]: nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path)) return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params) @@ -51,6 +57,23 @@ def _is_object_dict( return isinstance(value, dict) +def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrapper: + if _is_object_dict(details): + return PromptTokensDetailsWrapper(**details) + return PromptTokensDetailsWrapper( + cached_tokens=getattr(details, "cached_tokens", None), + audio_tokens=getattr(details, "audio_tokens", None), + text_tokens=getattr(details, "text_tokens", None), + image_tokens=getattr(details, "image_tokens", None), + cached_tokens_details=getattr(details, "cached_tokens_details", None), + video_tokens=getattr(details, "video_tokens", None), + cache_write_tokens=getattr(details, "cache_write_tokens", None), + cache_type=getattr(details, "cache_type", None), + web_search_requests=getattr(details, "web_search_requests", None), + google_maps_grounding_requests=getattr(details, "google_maps_grounding_requests", None), + ) + + def normalize_responses_api_stream_options( stream_options: object, ) -> ResponsesAPIStreamOptions | None: @@ -1047,7 +1070,7 @@ class ResponsesAPIRequestUtils: @staticmethod def _update_container_ids_in_response( - responses_api_response: ResponsesAPIResponse | dict[str, Any], + responses_api_response: ResponsesAPIResponse | dict[str, object], custom_llm_provider: str | None, litellm_metadata: dict[str, Any] | None = None, ) -> ResponsesAPIResponse | dict[str, object]: @@ -1061,15 +1084,16 @@ class ResponsesAPIRequestUtils: model_id: Final = model_info.get("id") # Get the output list - if isinstance(responses_api_response, dict): - output = responses_api_response.get("output", []) - else: - output = getattr(responses_api_response, "output", []) + output: Final = ( + responses_api_response.get("output", []) + if isinstance(responses_api_response, dict) + else attribute_or_none(responses_api_response, "output") + ) if not output: return responses_api_response - for item in output: + for item in iter_items(output): ResponsesAPIRequestUtils.encode_container_id_on_output_item( item=item, custom_llm_provider=custom_llm_provider, @@ -1244,27 +1268,11 @@ class ResponseAPILoggingUtils: response_api_usage = usage_input prompt_tokens: Final[int] = response_api_usage.input_tokens or 0 completion_tokens: Final[int] = response_api_usage.output_tokens or 0 - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if response_api_usage.input_tokens_details: - if isinstance(response_api_usage.input_tokens_details, dict): - prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), - audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), - text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), - image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), - video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), - cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), - cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), - web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), - google_maps_grounding_requests=getattr( - response_api_usage.input_tokens_details, "google_maps_grounding_requests", None - ), - ) + prompt_tokens_details: Final = ( + _prompt_tokens_details_wrapper(response_api_usage.input_tokens_details) + if response_api_usage.input_tokens_details + else None + ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None output_tokens_details: Final[OutputTokensDetails | None] = getattr( response_api_usage, "output_tokens_details", None @@ -1288,20 +1296,22 @@ class ResponseAPILoggingUtils: audio_tokens=audio_tokens, ) - extra_usage_fields: Final = { - key: value - for key, value in (response_api_usage.model_extra or {}).items() - if key - not in ( - "input_token_details", - "output_token_details", - "prompt_tokens", - "completion_tokens", - "total_tokens", - "prompt_tokens_details", - "completion_tokens_details", - ) - } + extra_usage_fields: Final = _UsageExtraFields( + **{ + key: value + for key, value in (as_str_mapping(response_api_usage.model_extra) or {}).items() + if key + not in ( + "input_token_details", + "output_token_details", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "prompt_tokens_details", + "completion_tokens_details", + ) + } + ) chat_usage: Final = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, diff --git a/litellm/router.py b/litellm/router.py index ac0933e158b..a55ab77faf7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -928,7 +928,7 @@ MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 def _retry_policy_ceiling(policy: RetryPolicy) -> int: """The most retries any error class under this policy can be granted.""" - retry_counts: Final[Iterable[object]] = policy.model_dump().values() + retry_counts: Final[tuple[object, ...]] = tuple(policy.model_dump().values()) return max((retries for retries in retry_counts if isinstance(retries, int)), default=0) diff --git a/litellm/utils.py b/litellm/utils.py index 14ee35e571e..7d61ad24ad9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4194,7 +4194,7 @@ def get_optional_params_embeddings( # retrieve all parameters passed to the function passed_params: Final = locals() - custom_llm_provider = passed_params.pop("custom_llm_provider", None) + passed_params.pop("custom_llm_provider", None) special_params: Final = passed_params.pop("kwargs") passed_params.pop("drop_params", None) @@ -4232,7 +4232,7 @@ def get_optional_params_embeddings( provider_config: BaseEmbeddingConfig | None = None optional_params = {} - if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): + if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_embedding_config( model=model, provider=LlmProviders(custom_llm_provider), @@ -10816,18 +10816,12 @@ def extract_duration_from_srt_or_vtt(srt_or_vtt_content: str) -> float | None: # Regular expression to match timestamps in the format "hh:mm:ss,ms" or "hh:mm:ss.ms" timestamp_pattern: Final = r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})" - timestamps: Final[Sequence[tuple[str, str, str, str]]] = re.findall(timestamp_pattern, srt_or_vtt_content) - - if not timestamps: - return None + timestamps: Final = re.finditer(timestamp_pattern, srt_or_vtt_content) # Convert timestamps to seconds and find the max (end time) - durations: Final = [] - match: tuple[str, str, str, str] - for match in timestamps: - hours, minutes, seconds, milliseconds = map(int, match) - total_seconds = hours * 3600 + minutes * 60 + seconds + milliseconds / 1000.0 - durations.append(total_seconds) + durations: Final = [ + int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3]) + int(match[4]) / 1000.0 for match in timestamps + ] return max(durations) if durations else None diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 8b33b1ce12c..b01a189ec32 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -5,7 +5,7 @@ LiteLLM SDK Functions for Creating and Searching Vector Stores import asyncio import builtins import contextvars -from collections.abc import Coroutine, Mapping +from collections.abc import Coroutine, Iterator, Mapping from functools import partial from typing import TYPE_CHECKING, Final @@ -53,6 +53,10 @@ def _direct_vector_store_embedding_executor( ) +def _items_without_key(values: Mapping[str, object], excluded_key: str) -> Iterator[tuple[str, object]]: + return ((key, value) for key, value in values.items() if key != excluded_key) + + def mock_vector_store_search_response( mock_results: builtins.list[VectorStoreSearchResult] | None = None, ): @@ -307,7 +311,7 @@ async def asearch( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: loop: Final = asyncio.get_event_loop() @@ -391,7 +395,7 @@ def search( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 487c00f735a..93385ea6618 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -42,7 +42,7 @@ IGNORE_FUNCTIONS = [ "contains_encrypted_marker", # max depth set. "_rotate_guardrail_row", # bounded by attempts_left. "_delete_nested_value_custom", # max depth set (bounded by number of path segments). - "filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. + "_filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. "__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion. "_validate_inheritance_chain", # max depth set (default 100) to prevent infinite recursion in policy inheritance validation. "_basic_json_schema_validate", # max depth set.