From 130699ca231f7e2409704a9cf22e59000a5f8d6b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:08:47 -0700 Subject: [PATCH] refactor(typing): clear 1143 basedpyright Any errors across 273 files (#46574) * refactor(typing): narrow Any sources in proxy, guardrails, bedrock and responses code Turns untyped json.loads, response.json(), header lookups and forwarded kwargs into object or precise types at their source, so downstream code type-checks without reportAny / reportExplicitAny. No runtime behavior changes. * refactor(typing): narrow Any sources in caching, MCP, providers and proxy request code Decoded JSON, cache reads and header lookups now enter as object or precise types, prisma tables used by the shadow-eval logger get typed Protocol views, and a few SDK parameters move from dict[str, Any] to Mapping[str, object]. No runtime behavior changes. * refactor(typing): narrow Any sources in proxy server, router, logging and shared utils * refactor(typing): narrow Any sources in provider transformations, integrations and proxy endpoints * refactor(typing): narrow Any sources across providers, proxy and router Deepens the Any cleanup on the hottest files: typed helpers in core_helpers for attribute and item access, identity TypeAdapter(object) wraps at untyped boundaries, explicit return annotations on shared provider and proxy helpers, and Protocols in place of Any parameters. No intended behavior change. --- .../proxy/auth/user_api_key_auth.py | 10 +- litellm/_logging.py | 17 +- litellm/_redis.py | 14 +- litellm/a2a_protocol/main.py | 10 +- litellm/a2a_protocol/streaming_iterator.py | 12 +- .../anthropic_interface/messages/__init__.py | 5 +- litellm/batches/batch_utils.py | 5 +- litellm/batches/main.py | 12 +- litellm/budget_manager.py | 9 +- litellm/caching/caching.py | 9 +- litellm/caching/gcs_cache.py | 8 +- litellm/caching/qdrant_semantic_cache.py | 11 +- litellm/caching/redis_cluster_cache.py | 8 +- litellm/caching/redis_semantic_cache.py | 9 +- litellm/caching/valkey_semantic_cache.py | 5 +- litellm/containers/endpoint_factory.py | 3 +- litellm/containers/utils.py | 3 +- litellm/google_genai/main.py | 120 ++++++----- litellm/google_genai/streaming_iterator.py | 6 +- .../harness/handlers/deepagents_handler.py | 15 +- .../anthropic_cache_control_hook.py | 7 +- litellm/integrations/arize/arize_phoenix.py | 10 +- .../arize/arize_phoenix_client.py | 13 +- .../bitbucket/bitbucket_client.py | 6 +- litellm/integrations/custom_guardrail.py | 20 +- litellm/integrations/custom_logger.py | 2 +- .../datadog/datadog_cost_management.py | 10 +- .../integrations/datadog/datadog_llm_obs.py | 5 +- .../gcs_bucket/gcs_bucket_base.py | 7 +- .../generic_api/generic_api_callback.py | 2 +- .../gitlab/gitlab_prompt_manager.py | 4 +- litellm/integrations/lago.py | 5 +- .../integrations/langfuse/langfuse_otel.py | 14 +- litellm/integrations/langsmith.py | 8 +- litellm/integrations/literal_ai.py | 4 +- litellm/integrations/lunary.py | 11 +- .../mavvrik_focus/mavvrik_focus_logger.py | 10 +- .../opentelemetry_utils/gen_ai_semconv.py | 3 +- litellm/integrations/otel/logger.py | 6 +- litellm/integrations/otel/model/metadata.py | 5 +- litellm/integrations/prometheus.py | 2 +- litellm/integrations/shadow_eval_logger.py | 60 +++++- .../integrations/vantage/vantage_logger.py | 8 +- .../websearch_interception/transformation.py | 21 +- litellm/interactions/agents/main.py | 20 +- litellm/interactions/http_handler.py | 13 +- litellm/interactions/main.py | 32 +-- litellm/litellm_core_utils/core_helpers.py | 39 +++- .../exception_mapping_utils.py | 15 +- .../litellm_core_utils/get_model_cost_map.py | 7 +- .../get_supported_openai_params.py | 6 +- .../json_validation_rule.py | 6 +- litellm/litellm_core_utils/litellm_logging.py | 78 +++---- .../convert_dict_to_response.py | 3 +- .../llm_response_utils/response_metadata.py | 2 +- .../logging_callback_manager.py | 2 +- .../prompt_templates/common_utils.py | 7 +- .../prompt_templates/factory.py | 15 +- .../huggingface_template_handler.py | 7 +- .../prompt_templates/image_handling.py | 11 +- .../litellm_core_utils/realtime_streaming.py | 10 +- litellm/litellm_core_utils/safe_json_loads.py | 8 +- .../specialty_caches/dynamic_logging_cache.py | 2 +- .../litellm_core_utils/streaming_handler.py | 11 +- .../chat/guardrail_translation/handler.py | 13 +- litellm/llms/anthropic/chat/handler.py | 6 +- litellm/llms/anthropic/chat/transformation.py | 13 +- litellm/llms/anthropic/common_utils.py | 5 +- .../pass_through/adapters/transformation.py | 4 +- .../pass_through/messages/handler.py | 8 +- .../messages/streaming_iterator.py | 5 +- .../pass_through/messages/transformation.py | 6 +- .../anthropic/pass_through/messages/utils.py | 16 +- litellm/llms/azure/realtime/handler.py | 12 +- .../llms/azure/responses/transformation.py | 5 +- .../anthropic/messages_transformation.py | 3 +- litellm/llms/base_llm/base_utils.py | 13 +- .../llms/base_llm/batches/transformation.py | 6 +- .../base_llm/containers/transformation.py | 8 +- litellm/llms/base_llm/files/transformation.py | 3 +- .../base_llm/google_genai/transformation.py | 10 +- .../llms/base_llm/managed_resources/utils.py | 7 +- .../llms/base_llm/responses/transformation.py | 3 +- .../vector_store_files/transformation.py | 6 +- .../llms/base_llm/videos/transformation.py | 5 +- litellm/llms/bedrock/base_aws_llm.py | 12 +- litellm/llms/bedrock/chat/converse_handler.py | 9 +- .../bedrock/chat/converse_transformation.py | 16 +- litellm/llms/bedrock/chat/invoke_handler.py | 7 +- .../amazon_deepseek_transformation.py | 4 +- ...mazon_twelvelabs_pegasus_transformation.py | 5 +- .../base_invoke_transformation.py | 8 +- .../bedrock/claude_platform/transformation.py | 9 +- ...n_nova_canvas_image_edit_transformation.py | 3 +- .../amazon_nova_canvas_transformation.py | 4 +- .../amazon_stability1_transformation.py | 6 +- .../anthropic_claude3_transformation.py | 4 +- .../bedrock/passthrough/transformation.py | 4 +- .../llms/bedrock/realtime/transformation.py | 9 +- .../bedrock_mantle/chat/transformation.py | 12 +- litellm/llms/bytez/chat/transformation.py | 3 +- litellm/llms/cohere/chat/v2_transformation.py | 5 +- litellm/llms/cohere/embed/transformation.py | 3 +- litellm/llms/custom_httpx/aiohttp_handler.py | 2 +- .../llms/custom_httpx/container_handler.py | 3 +- litellm/llms/custom_httpx/http_handler.py | 6 +- litellm/llms/custom_httpx/llm_http_handler.py | 14 +- .../llms/databricks/chat/transformation.py | 3 +- .../llms/deepagents/harness/transformation.py | 3 +- .../llms/deepseek/messages/transformation.py | 3 +- .../llms/deprecated_providers/aleph_alpha.py | 7 +- .../bytedance_transformation.py | 7 +- .../flux_pro_v11_transformation.py | 7 +- .../flux_schnell_transformation.py | 7 +- .../ideogram_v3_transformation.py | 7 +- .../llms/fireworks_ai/chat/transformation.py | 3 +- litellm/llms/gemini/count_tokens/handler.py | 2 +- .../gemini/google_genai/transformation.py | 4 +- .../gemini/vector_stores/transformation.py | 22 +- litellm/llms/gigachat/chat/transformation.py | 4 +- .../llms/gigachat/embedding/transformation.py | 3 +- .../embedding/transformation.py | 3 +- litellm/llms/groq/chat/transformation.py | 9 +- .../llms/hosted_vllm/rerank/transformation.py | 4 +- .../huggingface/embedding/transformation.py | 8 +- .../llms/jina_ai/embedding/transformation.py | 15 +- litellm/llms/langflow/a2a.py | 9 +- litellm/llms/lemonade/chat/transformation.py | 12 +- .../litellm_proxy/skills/prompt_injection.py | 7 +- .../ocr/guardrail_translation/handler.py | 5 +- .../llms/openai/chat/gpt_transformation.py | 5 +- .../chat/guardrail_translation/handler.py | 19 +- .../guardrail_translation/handler.py | 10 +- .../guardrail_translation/handler.py | 10 +- .../guardrail_translation/handler.py | 10 +- litellm/llms/openai/openai.py | 24 +-- .../llms/openai/responses/transformation.py | 6 +- .../speech/guardrail_translation/handler.py | 10 +- .../guardrail_translation/handler.py | 10 +- .../transcriptions/whisper_transformation.py | 7 +- .../guardrail_translation/handler.py | 34 ++- litellm/llms/petals/completion/handler.py | 8 +- litellm/llms/predibase/chat/transformation.py | 8 +- litellm/llms/replicate/chat/transformation.py | 3 +- litellm/llms/sagemaker/completion/handler.py | 11 +- .../sagemaker/completion/transformation.py | 7 +- litellm/llms/sap/chat/models.py | 33 +-- litellm/llms/sap/chat/transformation.py | 3 +- litellm/llms/sap/credentials.py | 5 +- .../vertex_ai/agent_engine/sse_iterator.py | 4 +- .../vertex_ai/agent_engine/transformation.py | 11 +- .../llms/vertex_ai/count_tokens/handler.py | 8 +- .../llms/vertex_ai/files/transformation.py | 2 +- litellm/llms/vertex_ai/fine_tuning/handler.py | 8 +- .../llms/vertex_ai/gemini/transformation.py | 22 +- .../vertex_and_google_ai_studio_gemini.py | 32 ++- .../vertex_ai/google_genai/transformation.py | 8 +- .../llms/vertex_ai/rag_engine/ingestion.py | 2 +- .../llms/vertex_ai/vertex_ai_non_gemini.py | 9 +- .../llama3/transformation.py | 7 +- .../volcengine/embedding/transformation.py | 3 +- litellm/llms/watsonx/embed/transformation.py | 19 +- litellm/main.py | 8 +- litellm/passthrough/main.py | 4 +- .../mcp_server/auth/user_api_key_auth_mcp.py | 3 +- .../mcp_server/discoverable_endpoints.py | 3 +- .../mcp_server/mcp_server_manager.py | 15 +- .../mcp_server/openapi_to_mcp_generator.py | 46 ++++- .../proxy/_experimental/mcp_server/server.py | 33 ++- .../proxy/_experimental/mcp_server/utils.py | 16 +- litellm/proxy/_lazy_features.py | 6 +- litellm/proxy/auth/auth_utils.py | 7 +- litellm/proxy/auth/handle_jwt.py | 6 +- litellm/proxy/client/cli/commands/agents.py | 6 +- litellm/proxy/client/cli/commands/auth.py | 6 +- .../client/cli/commands/autoroute/wizard.py | 3 +- .../proxy/client/cli/commands/encryption.py | 3 +- litellm/proxy/client/cli/commands/up.py | 6 +- litellm/proxy/client/cli/interface.py | 4 +- litellm/proxy/client/cli/main.py | 30 ++- litellm/proxy/client/models.py | 6 +- litellm/proxy/client/teams.py | 2 +- litellm/proxy/client/users.py | 14 +- litellm/proxy/common_request_processing.py | 31 ++- litellm/proxy/common_utils/debug_utils.py | 8 +- .../proxy/common_utils/http_parsing_utils.py | 4 +- litellm/proxy/common_utils/proxy_state.py | 6 +- litellm/proxy/db/base_client.py | 6 +- .../proxy/db/budget_window_spend_writer.py | 61 +++++- litellm/proxy/db/db_spend_update_writer.py | 4 +- litellm/proxy/db/health_check_latest.py | 19 +- litellm/proxy/db/prisma_client.py | 38 +++- litellm/proxy/db/proxy_worker_heartbeat.py | 18 +- litellm/proxy/db/routing_prisma_wrapper.py | 12 +- litellm/proxy/db/tool_registry_writer.py | 4 +- .../proxy/guardrails/guardrail_endpoints.py | 5 +- .../guardrail_hooks/azure/prompt_shield.py | 9 +- .../guardrail_hooks/bedrock_guardrails.py | 17 +- .../guardrail_hooks/compresr/compresr.py | 12 +- .../guardrail_hooks/dynamoai/dynamoai.py | 4 +- .../guardrail_hooks/headroom/headroom.py | 14 +- .../ibm_guardrails/ibm_detector.py | 6 +- .../litellm_content_filter/content_filter.py | 4 +- .../llm_shield_proxy/llm_shield_proxy.py | 25 +-- .../microsoft_purview/purview_dlp.py | 9 +- .../model_armor/model_armor.py | 16 +- .../panw_prisma_airs/panw_prisma_airs.py | 3 +- .../guardrails/guardrail_hooks/presidio.py | 60 +++--- .../guardrail_hooks/straiker/straiker.py | 24 ++- .../guardrail_hooks/tool_permission.py | 3 +- .../tool_policy/tool_policy_guardrail.py | 8 +- .../guardrail_hooks/xecguard/xecguard.py | 39 ++-- litellm/proxy/guardrails/usage_tracking.py | 15 +- litellm/proxy/hooks/batch_enqueued_tokens.py | 25 ++- .../proxy/hooks/parallel_request_limiter.py | 5 +- litellm/proxy/hooks/sensitive_data_routing.py | 25 ++- litellm/proxy/litellm_pre_call_utils.py | 11 +- .../config_override_endpoints.py | 5 +- .../credential_migration.py | 17 +- .../key_management_endpoints.py | 4 +- .../mcp_management_endpoints.py | 20 +- .../model_management_endpoints.py | 9 +- .../router_settings_endpoints.py | 3 +- .../team_callback_endpoints.py | 22 +- litellm/proxy/management_endpoints/ui_sso.py | 47 +++-- .../object_permission_utils.py | 19 +- .../in_flight_requests_middleware.py | 12 +- .../openai_files_endpoints/common_utils.py | 25 ++- .../llm_passthrough_endpoints.py | 3 +- .../gemini_passthrough_logging_handler.py | 23 +-- .../vertex_passthrough_logging_handler.py | 52 +++-- .../pass_through_endpoints.py | 15 +- litellm/proxy/policy_engine/init_policies.py | 44 ++-- litellm/proxy/proxy_server.py | 194 ++++++++++++------ litellm/proxy/realtime_endpoints/endpoints.py | 7 +- .../proxy/response_api_endpoints/endpoints.py | 6 +- .../proxy/response_polling/polling_handler.py | 10 +- .../spend_management_endpoints.py | 21 +- .../proxy_setting_endpoints.py | 12 +- litellm/proxy/utils.py | 23 ++- .../management_endpoints.py | 5 +- litellm/proxy/video_endpoints/utils.py | 5 +- litellm/realtime_api/main.py | 2 +- litellm/rerank_api/main.py | 6 +- litellm/responses/main.py | 6 +- .../mcp/litellm_proxy_mcp_handler.py | 19 +- .../responses/mcp/mcp_streaming_iterator.py | 5 + litellm/router.py | 40 ++-- .../adaptive_router/adaptive_router.py | 6 +- .../auto_router/auto_router.py | 2 +- .../auto_router/litellm_encoder.py | 4 +- .../capability_classifier.py | 2 +- .../complexity_router/complexity_router.py | 6 +- .../complexity_router/stall_detector.py | 5 +- litellm/router_strategy/lowest_tpm_rpm_v2.py | 2 +- .../quality_router/quality_router.py | 12 +- litellm/router_utils/batch_utils.py | 10 +- litellm/router_utils/common_utils.py | 3 +- litellm/router_utils/cooldown_handlers.py | 2 +- litellm/router_utils/handle_error.py | 2 +- litellm/router_utils/health_state_cache.py | 2 +- .../deployment_affinity_check.py | 6 +- .../pre_call_checks/model_rate_limit_check.py | 2 +- litellm/sandbox/main.py | 4 +- litellm/search/main.py | 8 +- litellm/secret_managers/aws_secret_manager.py | 2 +- .../get_azure_ad_token_provider.py | 9 +- litellm/skills/main.py | 4 +- .../guardrails/guardrail_hooks/dynamoai.py | 3 +- litellm/utils.py | 33 +-- litellm/vector_store_files/main.py | 14 +- litellm/vector_stores/utils.py | 6 +- 272 files changed, 2092 insertions(+), 1160 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py index dc9fdeb78e2..d73431d1e0f 100644 --- a/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py +++ b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py @@ -1,14 +1,16 @@ -from typing import Any, Optional +from collections.abc import Awaitable, Callable from fastapi import Request from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ProxyException async def enterprise_custom_auth( - request: Request, api_key: str, user_custom_auth: Optional[Any] -) -> Optional[UserAPIKeyAuth]: + request: Request, + api_key: str, + user_custom_auth: Callable[[Request, str], Awaitable[object]] | None, +) -> object: from litellm_enterprise.proxy.proxy_server import custom_auth_settings if user_custom_auth is None: diff --git a/litellm/_logging.py b/litellm/_logging.py index be6bcf476f7..913e9a20101 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -6,12 +6,14 @@ import logging import os import re import sys -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from datetime import datetime from logging import Formatter from typing import Final, TextIO from urllib.parse import unquote +from pydantic import TypeAdapter + import litellm from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, @@ -84,6 +86,7 @@ _redact_string = redact_string _REDACTED_RECORD_ATTR: Final = "litellm_redacted" _REDACTED_STAMP: Final = object() +_DECODED_LOG_PAYLOAD: Final = TypeAdapter(object) _UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None)) @@ -91,6 +94,10 @@ def _is_redacted(record: logging.LogRecord) -> bool: return getattr(record, _REDACTED_RECORD_ATTR, None) is _REDACTED_STAMP +def _record_attributes(record: logging.LogRecord) -> Mapping[str, object]: + return record.__dict__ + + def _scrubbing_changed_nothing(scrubbed: object, original: object) -> bool: try: return bool(scrubbed == original) @@ -438,7 +445,7 @@ def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: key, substituted_color if key == "color_message" and substituted_color is not None else value ), ) - for key, value in record.__dict__.items() + for key, value in _record_attributes(record).items() if key not in _STANDARD_RECORD_ATTRS and key != _REDACTED_RECORD_ATTR and not isinstance(value, _UNREDACTED_SCALAR_TYPES) @@ -669,7 +676,7 @@ def _try_parse_json_message(message: str) -> dict[str, object] | None: msg_stripped: Final = message.strip() if not (msg_stripped.startswith("{") or msg_stripped.startswith("[")): return None - parsed: Final[object] = safe_json_loads(message, default=None) + parsed: Final = _DECODED_LOG_PAYLOAD.validate_python(safe_json_loads(message, default=None)) if parsed is None or not isinstance(parsed, dict): return None return parsed @@ -699,7 +706,7 @@ def _try_parse_embedded_python_dict(message: str) -> dict[str, object] | None: if depth == 0: substr = message[start : j + 1] try: - result: object = ast.literal_eval(substr) + result: object = _DECODED_LOG_PAYLOAD.validate_python(ast.literal_eval(substr)) if isinstance(result, dict) and len(result) > 0: return result except (ValueError, SyntaxError, TypeError): @@ -757,7 +764,7 @@ class JsonFormatter(Formatter): json_record[key] = value # Include extra attributes passed via logger.debug("msg", extra={...}) - for key, value in record.__dict__.items(): + for key, value in _record_attributes(record).items(): if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record: json_record[key] = value diff --git a/litellm/_redis.py b/litellm/_redis.py index 186f8523519..96777887c74 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -19,6 +19,7 @@ from urllib.parse import urlsplit, urlunsplit import redis import redis.asyncio as async_redis +from pydantic import TypeAdapter from redis.credentials import CredentialProvider from litellm import get_secret, get_secret_str @@ -40,6 +41,7 @@ from ._logging import verbose_logger _generate_gcp_iam_access_token = generate_gcp_iam_access_token AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" +_UNWRAPPED_INIT: Final = TypeAdapter(object) _AWS_IAM_KWARG_NAMES: Final = ( "aws_iam_auth", @@ -64,7 +66,7 @@ def _unwrapped_init_args(cls: type) -> frozenset[str]: ``@deprecated_args`` too, which the same class of bug as ``_init_arg_names`` would otherwise silently empty this allowlist through (see its docstring). """ - spec: Final = inspect.getfullargspec(inspect.unwrap(cls.__init__)) + spec: Final = inspect.getfullargspec(_UNWRAPPED_INIT.validate_python(inspect.unwrap(cls.__init__))) return frozenset(spec.args + spec.kwonlyargs) @@ -112,7 +114,7 @@ def _init_arg_names(cls: type) -> frozenset[str]: name for klass in inspect.getmro(cls) if klass is not object - for spec in (inspect.getfullargspec(inspect.unwrap(klass.__init__)),) + for spec in (inspect.getfullargspec(_UNWRAPPED_INIT.validate_python(inspect.unwrap(klass.__init__))),) for name in spec.args + spec.kwonlyargs ) @@ -935,6 +937,10 @@ def get_redis_connection_pool( return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs) +def _mask_for_display(masker: SensitiveDataMasker, redis_kwargs: Mapping[str, object]) -> Mapping[str, object]: + return masker.mask_dict(redis_kwargs) + + def _redis_kwargs_for_logging(redis_kwargs: Mapping[str, object]) -> Mapping[str, object]: return { key: "" @@ -966,7 +972,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None: masker = SensitiveDataMasker() # Mask sensitive data in redis_kwargs - masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging) + masked_redis_kwargs = _mask_for_display(masker, redis_kwargs_for_logging) # Create main panel title title: Final = Text("Redis Configuration", style="bold blue") @@ -1029,7 +1035,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None: except ImportError: # Fallback to simple logging if rich is not available masker = SensitiveDataMasker() - masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging) + masked_redis_kwargs = _mask_for_display(masker, redis_kwargs_for_logging) verbose_logger.info("Redis configuration: %s", masked_redis_kwargs) except Exception as e: verbose_logger.error("Error pretty printing Redis configuration: %s", e) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 4de2ff49b56..e5bd38bb7ed 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -14,7 +14,9 @@ import datetime import uuid from collections.abc import AsyncIterator, Coroutine, Mapping from types import MappingProxyType, ModuleType -from typing import TYPE_CHECKING, Any, Final, Optional, cast +from typing import TYPE_CHECKING, Final, Optional, cast + +from typing_extensions import TypedDict, Unpack import litellm from litellm._logging import verbose_logger, verbose_proxy_logger @@ -89,6 +91,10 @@ from litellm.a2a_protocol.exceptions import A2ALocalhostURLError A2ACardResolver: Final = LiteLLMA2ACardResolver +class _SendMessageOptions(TypedDict, total=False, extra_items=object): + pass + + def _set_usage_on_logging_obj( kwargs: Mapping[str, object], prompt_tokens: int, @@ -563,7 +569,7 @@ async def asend_message( def send_message( a2a_client: "A2AClientType", request: "SendMessageRequest", - **kwargs: Any, + **kwargs: Unpack[_SendMessageOptions], ) -> LiteLLMSendMessageResponse | Coroutine[object, object, LiteLLMSendMessageResponse]: """ Sync: Send a message to an A2A agent. diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 8232d7cf2d8..0b649f62a66 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -7,16 +7,21 @@ from collections.abc import AsyncIterator from datetime import datetime from typing import TYPE_CHECKING, Final +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.a2a_protocol.cost_calculator import A2ACostCalculator from litellm.a2a_protocol.utils import A2ARequestUtils from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj if TYPE_CHECKING: from a2a.compat.v0_3.types import SendStreamingMessageRequest, SendStreamingMessageResponse +_OPAQUE_PAYLOAD: Final = TypeAdapter(object) + class A2AStreamingIterator: """ @@ -160,8 +165,11 @@ class A2AStreamingIterator: # Add final chunk result if available if self.final_chunk: try: - chunk_dict: Final = self.final_chunk.model_dump(mode="json", exclude_none=True) - result["result"] = chunk_dict.get("result", {}) + chunk_dict: Final = as_str_mapping( + _OPAQUE_PAYLOAD.validate_python(self.final_chunk.model_dump(mode="json", exclude_none=True)) + ) + if chunk_dict is not None: + result["result"] = chunk_dict.get("result", {}) except Exception: pass diff --git a/litellm/anthropic_interface/messages/__init__.py b/litellm/anthropic_interface/messages/__init__.py index 30319104844..4c5d59d0fcb 100644 --- a/litellm/anthropic_interface/messages/__init__.py +++ b/litellm/anthropic_interface/messages/__init__.py @@ -11,7 +11,6 @@ This is an __init__.py file to allow the following interface """ from collections.abc import AsyncIterator, Coroutine, Iterator -from typing import Any from litellm.messages import ( anthropic_messages as _async_anthropic_messages, @@ -102,8 +101,8 @@ def create( ) -> ( AnthropicMessagesResponse | Iterator[bytes] - | AsyncIterator[Any] - | Coroutine[Any, Any, AnthropicMessagesResponse | AsyncIterator[Any] | Iterator[bytes]] + | AsyncIterator[object] + | Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[object] | Iterator[bytes]] ): """ Async wrapper for Anthropic's messages API diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 555c14ae6c9..e7b009eaefe 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -5,6 +5,8 @@ from dataclasses import replace as dataclasses_replace from enum import Enum from typing import Any, Final, Literal, cast +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS @@ -35,6 +37,7 @@ class BatchCostUsageResult: _COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"}) +_DECODED_BATCH_LINE: Final = TypeAdapter(object) def _uses_native_vertex_output( @@ -624,7 +627,7 @@ def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict[str, object def parse_batch_output_line(line: bytes) -> dict[str, object] | None: try: - parsed: Final[object] = json.loads(line) + parsed: Final = _DECODED_BATCH_LINE.validate_python(json.loads(line)) except ValueError as e: verbose_logger.warning("skipping malformed batch output line: %s", str(e)) return None diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 548cf693ab9..1acd7ad67d1 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -127,7 +127,7 @@ async def acreate_batch( metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, - output_expires_after: dict[str, Any] | None = None, + output_expires_after: FileExpiresAfter | None = None, **kwargs, ) -> LiteLLMBatch: """ @@ -191,9 +191,9 @@ def create_batch( metadata: dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, - output_expires_after: dict[str, Any] | None = None, + output_expires_after: FileExpiresAfter | None = None, **kwargs, -) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: +) -> LiteLLMBatch | Coroutine[object, None, LiteLLMBatch]: """ Creates and executes a batch from an uploaded file of request @@ -246,7 +246,7 @@ def create_batch( extra_body=extra_body, ) if output_expires_after is not None: - _create_batch_request["output_expires_after"] = cast(FileExpiresAfter, output_expires_after) + _create_batch_request["output_expires_after"] = output_expires_after if model is not None: provider_config = ProviderConfigManager.get_provider_batches_config( model=model, @@ -568,7 +568,7 @@ def retrieve_batch( extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, **kwargs, -) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: +) -> LiteLLMBatch | Coroutine[object, None, LiteLLMBatch]: """ Retrieves a batch. @@ -941,7 +941,7 @@ def cancel_batch( extra_headers: dict[str, str] | None = None, extra_body: dict[str, str] | None = None, **kwargs, -) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: +) -> LiteLLMBatch | Coroutine[object, None, LiteLLMBatch]: """ Cancels a batch. diff --git a/litellm/budget_manager.py b/litellm/budget_manager.py index dcb5a7cc183..0f8ec4be036 100644 --- a/litellm/budget_manager.py +++ b/litellm/budget_manager.py @@ -13,6 +13,8 @@ import threading import time from typing import Final, Literal +from pydantic import TypeAdapter + import litellm from litellm.constants import ( DAYS_IN_A_MONTH, @@ -22,6 +24,8 @@ from litellm.constants import ( ) from litellm.utils import ModelResponse +_BUDGET_SERVER_REPLY: Final = TypeAdapter(object) + class BudgetManager: def __init__( @@ -211,6 +215,5 @@ class BudgetManager: elif self.client_type == "hosted": url: Final = self.api_base + "/set_budget" data: Final = {"project_name": self.project_name, "user_dict": self.user_dict} - response = litellm.module_level_client.post(url, headers=self.headers, json=data) - response = response.json() - return response + response: Final = litellm.module_level_client.post(url, headers=self.headers, json=data) + return _BUDGET_SERVER_REPLY.validate_python(response.json()) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 1d101b65e36..163fbd64a9d 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -18,7 +18,7 @@ from contextlib import contextmanager from enum import Enum from typing import Any, Final, Literal, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._internal_context import current_service_target, service_target @@ -42,6 +42,7 @@ from .redis_semantic_cache import RedisSemanticCache from .s3_cache import S3Cache RESPONSE_CACHE_TARGET: Final = "llm_response" +_DECODED_LITERAL: Final = TypeAdapter(object) @contextmanager @@ -599,9 +600,11 @@ class Cache: if isinstance(cached_response, dict): pass else: - cached_response = json.loads(cached_response) # Convert string to dictionary + cached_response = _DECODED_LITERAL.validate_python( + json.loads(cached_response) + ) # Convert string to dictionary except Exception: - cached_response = ast.literal_eval(cached_response) + cached_response = _DECODED_LITERAL.validate_python(ast.literal_eval(cached_response)) return cached_response return cached_result diff --git a/litellm/caching/gcs_cache.py b/litellm/caching/gcs_cache.py index 76fcba86eb0..7b3003dbbc6 100644 --- a/litellm/caching/gcs_cache.py +++ b/litellm/caching/gcs_cache.py @@ -7,6 +7,8 @@ import json from typing import Final from urllib.parse import quote +from pydantic import TypeAdapter + from litellm._logging import print_verbose, verbose_logger from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase from litellm.llms.custom_httpx.http_handler import ( @@ -17,6 +19,8 @@ from litellm.llms.custom_httpx.http_handler import ( from .base_cache import BaseCache +_DECODED_JSON: Final = TypeAdapter(object) + class GCSCache(BaseCache): def __init__( @@ -70,7 +74,7 @@ class GCSCache(BaseCache): url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{quote(object_name, safe='')}?alt=media" response: Final = self.sync_client.get(url=url, headers=headers) if response.status_code == 200: - cached_response: Final = json.loads(response.text) + cached_response: Final = _DECODED_JSON.validate_python(json.loads(response.text)) verbose_logger.debug( "Got GCS Cache: key: %s, cached_response %s. Type Response %s", key, @@ -90,7 +94,7 @@ class GCSCache(BaseCache): url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{quote(object_name, safe='')}?alt=media" response: Final = await self.async_client.get(url=url, headers=headers) if response.status_code == 200: - return json.loads(response.text) + return _DECODED_JSON.validate_python(json.loads(response.text)) return None except Exception as e: verbose_logger.error("GCS Caching: async_get_cache() - Got exception from GCS: %s", e) diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index eb88df066f6..924506dfa45 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -15,6 +15,8 @@ import os from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from pydantic import TypeAdapter + import litellm from litellm._logging import print_verbose from litellm.constants import ( @@ -38,6 +40,7 @@ from ._embedding_router import ( from .base_cache import BaseCache _WAIT_FOR_INDEXING: Final = MappingProxyType({"wait": "true"}) +_EMBEDDING_VECTOR: Final = TypeAdapter(object) if TYPE_CHECKING: from litellm.router import Router @@ -295,7 +298,7 @@ class QdrantSemanticCache(BaseCache): ) # get the embedding - embedding: Final = embedding_response["data"][0]["embedding"] + embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) value = str(value) assert isinstance(value, str) @@ -334,7 +337,7 @@ class QdrantSemanticCache(BaseCache): ) # get the embedding - embedding: Final = embedding_response["data"][0]["embedding"] + embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) data: Final = { "vector": embedding, @@ -404,7 +407,7 @@ class QdrantSemanticCache(BaseCache): embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding - embedding: Final = embedding_response["data"][0]["embedding"] + embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) value = str(value) assert isinstance(value, str) @@ -440,7 +443,7 @@ class QdrantSemanticCache(BaseCache): embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding - embedding: Final = embedding_response["data"][0]["embedding"] + embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"]) data: Final = { "vector": embedding, diff --git a/litellm/caching/redis_cluster_cache.py b/litellm/caching/redis_cluster_cache.py index 5e926fd2ed8..6fb84df331e 100644 --- a/litellm/caching/redis_cluster_cache.py +++ b/litellm/caching/redis_cluster_cache.py @@ -9,7 +9,7 @@ import asyncio import contextvars import os import weakref -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar @@ -24,7 +24,7 @@ if TYPE_CHECKING: pipeline = Pipeline async_redis_client = Redis - Span = _Span | Any + Span = _Span else: pipeline = Any async_redis_client = Any @@ -126,13 +126,13 @@ class RedisClusterCache(RedisCache): return call() return self._connection_gate.run(call, self._get_async_client_cache_key()) - def _run_redis_mget_operation(self, keys: list[str]) -> list[Any]: + def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ Overrides `_run_redis_mget_operation` in redis_cache.py """ return self.redis_client.mget_nonatomic(keys=keys) - async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]: + async def _async_run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: """ Overrides `_async_run_redis_mget_operation` in redis_cache.py """ diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 8f99d76ba46..b892e4c3a1d 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -117,7 +117,8 @@ class RedisSemanticCache(BaseCache): password = password or os.environ["REDIS_PASSWORD"] except KeyError as e: # Raise a more informative exception if any of the required keys are missing - missing_var: Final[object] = e.args[0] + key_error_args: Final[tuple[object, ...]] = e.args + missing_var: Final = key_error_args[0] raise ValueError( f"Missing required Redis configuration: {missing_var}. Provide {missing_var} or redis_url." ) from e @@ -310,9 +311,9 @@ class RedisSemanticCache(BaseCache): return return - content = getattr(value, "content", None) - if content is not None: - cls._collect_responses_input_text(content, prompt_parts) + attribute_content: Final[object] = getattr(value, "content", None) + if attribute_content is not None: + cls._collect_responses_input_text(attribute_content, prompt_parts) return for text_key in ("text", "output", "input_text", "output_text"): diff --git a/litellm/caching/valkey_semantic_cache.py b/litellm/caching/valkey_semantic_cache.py index 5a3f9a4184c..645bd0892f4 100644 --- a/litellm/caching/valkey_semantic_cache.py +++ b/litellm/caching/valkey_semantic_cache.py @@ -21,6 +21,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any, Final +from pydantic import TypeAdapter from redis import Redis from redis.asyncio import Redis as AsyncRedis from redis.commands.search.field import TagField, VectorField @@ -34,6 +35,8 @@ from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector from ._embedding_router import resolve_embedding_timeout from .redis_semantic_cache import RedisSemanticCache +_OPAQUE_SEARCH_FIELD: Final = TypeAdapter(object) + @dataclass(frozen=True, slots=True) class _ValkeyCacheHit: @@ -226,7 +229,7 @@ class ValkeySemanticCache(RedisSemanticCache): if not docs: return None doc: Final = docs[0] - response_field: Final[object] = getattr(doc, cls.RESPONSE_FIELD_NAME) + response_field: Final = _OPAQUE_SEARCH_FIELD.validate_python(getattr(doc, cls.RESPONSE_FIELD_NAME)) distance_field: Final[str | bytes | float] = getattr(doc, cls.DISTANCE_FIELD_NAME) return _ValkeyCacheHit( response=str(response_field), diff --git a/litellm/containers/endpoint_factory.py b/litellm/containers/endpoint_factory.py index a969a9cb877..a4008bedf01 100644 --- a/litellm/containers/endpoint_factory.py +++ b/litellm/containers/endpoint_factory.py @@ -36,6 +36,7 @@ RESPONSE_TYPES: Final[dict[str, type]] = { "DeleteContainerFileResponse": DeleteContainerFileResponse, } _ENDPOINTS_DOCUMENT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) +_ENDPOINT_RESULT: Final = TypeAdapter(object) def _load_endpoints_config() -> dict: @@ -173,7 +174,7 @@ def create_async_endpoint_function( init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = _ENDPOINT_RESULT.validate_python(await init_response) else: response = init_response diff --git a/litellm/containers/utils.py b/litellm/containers/utils.py index 44923719543..16d54766e96 100644 --- a/litellm/containers/utils.py +++ b/litellm/containers/utils.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final, TypeVar from litellm.llms.base_llm.containers.transformation import BaseContainerConfig @@ -99,7 +100,7 @@ class ContainerRequestUtils: response_obj: T, custom_llm_provider: str | None, litellm_metadata: dict[str, Any] | None = None, - extra_body: dict[str, Any] | None = None, + extra_body: Mapping[str, object] | None = None, ) -> T: """ Encode container_id in response object with provider/model metadata for routing. diff --git a/litellm/google_genai/main.py b/litellm/google_genai/main.py index f74d78aeac1..beb4b96d787 100644 --- a/litellm/google_genai/main.py +++ b/litellm/google_genai/main.py @@ -1,11 +1,10 @@ import asyncio import contextvars -from collections.abc import Iterator from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, Final import httpx -from pydantic import ConfigDict +from pydantic import ConfigDict, TypeAdapter import litellm from litellm.constants import request_timeout @@ -39,6 +38,7 @@ else: ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() +_GENERATE_CONTENT_RESULT: Final = TypeAdapter(object) ################################################# @@ -248,7 +248,7 @@ async def agenerate_content( # LiteLLM specific params, custom_llm_provider: str | None = None, **kwargs, -) -> Any: +) -> object: """ Async: Generate content using Google GenAI """ @@ -286,7 +286,7 @@ async def agenerate_content( init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = _GENERATE_CONTENT_RESULT.validate_python(await init_response) else: response = init_response @@ -316,7 +316,7 @@ def generate_content( # LiteLLM specific params, custom_llm_provider: str | None = None, **kwargs, -) -> Any: +) -> object: """ Generate content using Google GenAI """ @@ -362,22 +362,24 @@ def generate_content( ) # Call the standard handler - response: Final = base_llm_http_handler.generate_content_handler( - model=setup_result.model, - contents=contents, - tools=tools, - generate_content_provider_config=setup_result.generate_content_provider_config, - generate_content_config_dict=setup_result.generate_content_config_dict, - custom_llm_provider=setup_result.custom_llm_provider, - litellm_params=setup_result.litellm_params, - logging_obj=setup_result.litellm_logging_obj, - extra_headers=extra_headers, - extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), - timeout=timeout or request_timeout, - _is_async=_is_async, - client=kwargs.get("client"), - litellm_metadata=kwargs.get("litellm_metadata", {}), - system_instruction=system_instruction, + response: Final = _GENERATE_CONTENT_RESULT.validate_python( + base_llm_http_handler.generate_content_handler( + model=setup_result.model, + contents=contents, + tools=tools, + generate_content_provider_config=setup_result.generate_content_provider_config, + generate_content_config_dict=setup_result.generate_content_config_dict, + custom_llm_provider=setup_result.custom_llm_provider, + litellm_params=setup_result.litellm_params, + logging_obj=setup_result.litellm_logging_obj, + extra_headers=extra_headers, + extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + litellm_metadata=kwargs.get("litellm_metadata", {}), + system_instruction=system_instruction, + ) ) return response @@ -406,7 +408,7 @@ async def agenerate_content_stream( # LiteLLM specific params, custom_llm_provider: str | None = None, **kwargs, -) -> Any: +) -> object: """ Async: Generate content using Google GenAI with streaming response """ @@ -457,23 +459,25 @@ async def agenerate_content_stream( # Call the handler with async enabled and streaming # Return the coroutine directly for the router to handle - return await base_llm_http_handler.generate_content_handler( - model=setup_result.model, - contents=contents, - generate_content_provider_config=setup_result.generate_content_provider_config, - generate_content_config_dict=setup_result.generate_content_config_dict, - tools=tools, - custom_llm_provider=setup_result.custom_llm_provider, - litellm_params=setup_result.litellm_params, - logging_obj=setup_result.litellm_logging_obj, - extra_headers=extra_headers, - extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), - timeout=timeout or request_timeout, - _is_async=True, - client=kwargs.get("client"), - stream=True, - litellm_metadata=kwargs.get("litellm_metadata", {}), - system_instruction=system_instruction, + return _GENERATE_CONTENT_RESULT.validate_python( + await base_llm_http_handler.generate_content_handler( + model=setup_result.model, + contents=contents, + generate_content_provider_config=setup_result.generate_content_provider_config, + generate_content_config_dict=setup_result.generate_content_config_dict, + tools=tools, + custom_llm_provider=setup_result.custom_llm_provider, + litellm_params=setup_result.litellm_params, + logging_obj=setup_result.litellm_logging_obj, + extra_headers=extra_headers, + extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), + timeout=timeout or request_timeout, + _is_async=True, + client=kwargs.get("client"), + stream=True, + litellm_metadata=kwargs.get("litellm_metadata", {}), + system_instruction=system_instruction, + ) ) except Exception as e: @@ -501,7 +505,7 @@ def generate_content_stream( # LiteLLM specific params, custom_llm_provider: str | None = None, **kwargs, -) -> Iterator[Any]: +) -> object: """ Generate content using Google GenAI with streaming response """ @@ -546,23 +550,25 @@ def generate_content_stream( ) # Call the handler with streaming enabled (sync version) - return base_llm_http_handler.generate_content_handler( - model=setup_result.model, - contents=contents, - generate_content_provider_config=setup_result.generate_content_provider_config, - generate_content_config_dict=setup_result.generate_content_config_dict, - tools=tools, - custom_llm_provider=setup_result.custom_llm_provider, - litellm_params=setup_result.litellm_params, - logging_obj=setup_result.litellm_logging_obj, - extra_headers=extra_headers, - extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), - timeout=timeout or request_timeout, - _is_async=_is_async, - client=kwargs.get("client"), - stream=True, - litellm_metadata=kwargs.get("litellm_metadata", {}), - system_instruction=system_instruction, + return _GENERATE_CONTENT_RESULT.validate_python( + base_llm_http_handler.generate_content_handler( + model=setup_result.model, + contents=contents, + generate_content_provider_config=setup_result.generate_content_provider_config, + generate_content_config_dict=setup_result.generate_content_config_dict, + tools=tools, + custom_llm_provider=setup_result.custom_llm_provider, + litellm_params=setup_result.litellm_params, + logging_obj=setup_result.litellm_logging_obj, + extra_headers=extra_headers, + extra_body=_merge_native_request_fields(setup_result.native_request_fields, extra_body), + timeout=timeout or request_timeout, + _is_async=_is_async, + client=kwargs.get("client"), + stream=True, + litellm_metadata=kwargs.get("litellm_metadata", {}), + system_instruction=system_instruction, + ) ) except Exception as e: diff --git a/litellm/google_genai/streaming_iterator.py b/litellm/google_genai/streaming_iterator.py index 64508ec93a6..37d08dfb59a 100644 --- a/litellm/google_genai/streaming_iterator.py +++ b/litellm/google_genai/streaming_iterator.py @@ -67,7 +67,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator: request_body: dict, model: str, custom_llm_provider: str, - hidden_params: dict[str, Any] | None = None, + hidden_params: dict[str, object] | None = None, ): self.litellm_logging_obj = litellm_logging_obj self.request_body = request_body @@ -118,7 +118,7 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent litellm_metadata: dict, custom_llm_provider: str, request_body: dict | None = None, - hidden_params: dict[str, Any] | None = None, + hidden_params: dict[str, object] | None = None, ): super().__init__( litellm_logging_obj=logging_obj, @@ -169,7 +169,7 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo litellm_metadata: dict, custom_llm_provider: str, request_body: dict | None = None, - hidden_params: dict[str, Any] | None = None, + hidden_params: dict[str, object] | None = None, ): super().__init__( litellm_logging_obj=logging_obj, diff --git a/litellm/harness/handlers/deepagents_handler.py b/litellm/harness/handlers/deepagents_handler.py index 5215562a1c5..102f4a741cf 100644 --- a/litellm/harness/handlers/deepagents_handler.py +++ b/litellm/harness/handlers/deepagents_handler.py @@ -13,8 +13,8 @@ import asyncio import importlib from collections.abc import AsyncIterator, Callable, Mapping from dataclasses import dataclass -from types import MappingProxyType, ModuleType -from typing import TYPE_CHECKING, Any +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Protocol from litellm.harness.context import SessionContext from litellm.harness.errors import HarnessError, HarnessInstallFailed @@ -51,10 +51,17 @@ if TYPE_CHECKING: from langgraph.types import Command from litellm.llms.base_llm.harness.transformation import BaseHarnessConfig + from litellm.llms.deepagents.harness.sandbox_backend import SandboxBackend, ToolFilterMiddleware, UsageCallback _MODEL_NODE = "model" +class _SandboxBackendModule(Protocol): + SandboxBackend: type[SandboxBackend] + UsageCallback: type[UsageCallback] + ToolFilterMiddleware: type[ToolFilterMiddleware] + + @dataclass(frozen=True) class DeepAgentsDeps: """The optional-dependency entrypoints this handler uses.""" @@ -65,7 +72,7 @@ class DeepAgentsDeps: command_cls: Callable[..., Command] subagent_defaults: Mapping[str, object] convert_to_openai_messages: Any - backend: ModuleType + backend: _SandboxBackendModule def load_deps() -> DeepAgentsDeps: @@ -77,7 +84,7 @@ def load_deps() -> DeepAgentsDeps: memory = importlib.import_module("langgraph.checkpoint.memory") lg_types = importlib.import_module("langgraph.types") messages = importlib.import_module("langchain_core.messages") - backend = importlib.import_module("litellm.llms.deepagents.harness.sandbox_backend") + from litellm.llms.deepagents.harness import sandbox_backend as backend except ImportError as e: raise HarnessInstallFailed(f"{INSTALL_HINT} ({e})") from e return DeepAgentsDeps( diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index c2958f1e120..05f237394e0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -75,6 +75,7 @@ OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object]) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues @@ -216,7 +217,7 @@ def _carries_cache_breakpoint(block: object) -> bool: def _attribute_or_key(value: object, key: str) -> object | None: if hasattr(value, key): - return getattr(value, key) + return _ATTRIBUTE_VALUE.validate_python(getattr(value, key)) if isinstance(value, Mapping): return value.get(key) return None @@ -983,7 +984,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def maybe_seed_default_injection_points( - non_default_params: dict[str, Any], + non_default_params: dict[str, object], messages: list[AllMessageValues], model: str, custom_llm_provider: str | None, @@ -1109,7 +1110,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): def maybe_inject_cache_control( messages: list[dict], system: str | list | None, - kwargs: dict[str, Any], + kwargs: dict[str, object], model: str | None = None, custom_llm_provider: str | None = None, tools: list[dict] | None = None, diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index 5b52c59cae2..ce585ad26c4 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -1,6 +1,7 @@ import os import threading from collections import OrderedDict +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_logger @@ -22,7 +23,7 @@ if TYPE_CHECKING: Protocol = _Protocol OpenTelemetryConfig = _OpenTelemetryConfig - Span = _Span | Any + Span = _Span OpenTelemetry = _OpenTelemetry LITELLM_TRACER_NAME: str else: @@ -100,7 +101,7 @@ class ArizePhoenixLogger(OpenTelemetry): if getattr(self, "_use_injected_tracer_provider", False): return - shared_processor: Final = getattr(self, "_shared_span_processor", None) + shared_processor: Final[SpanProcessor | None] = getattr(self, "_shared_span_processor", None) if shared_processor is not None: try: shared_processor.force_flush() @@ -111,7 +112,8 @@ class ArizePhoenixLogger(OpenTelemetry): ) with getattr(self, "_project_providers_lock", threading.Lock()): - providers: Final = list(getattr(self, "_project_providers", {}).values()) + project_providers: Final[Mapping[str, TracerProvider]] = getattr(self, "_project_providers", {}) + providers: Final = list(project_providers.values()) for provider in providers: try: @@ -193,7 +195,7 @@ class ArizePhoenixLogger(OpenTelemetry): primary ``otel`` callback which handles proxy-level parent spans. """ - def set_attributes(self, span: Span, kwargs, response_obj: Any | None): + def set_attributes(self, span: Span, kwargs, response_obj: object): ArizePhoenixLogger.set_arize_phoenix_attributes(span, kwargs, response_obj) @staticmethod diff --git a/litellm/integrations/arize/arize_phoenix_client.py b/litellm/integrations/arize/arize_phoenix_client.py index b63cacede03..6e8ebb99a9e 100644 --- a/litellm/integrations/arize/arize_phoenix_client.py +++ b/litellm/integrations/arize/arize_phoenix_client.py @@ -68,7 +68,7 @@ class ArizePhoenixClient: try: # Use the underlying httpx client directly to avoid query param extraction - response = self.http_handler.get(url, headers=self.headers) + response: Final = self.http_handler.get(url, headers=self.headers) response.raise_for_status() data: Final = response.json() @@ -76,15 +76,16 @@ class ArizePhoenixClient: except Exception as e: # Check if it's an HTTP error - response = getattr(e, "response", None) - if response is not None and hasattr(response, "status_code"): - if response.status_code == 404: + error_response: Final[object] = getattr(e, "response", None) + if error_response is not None and hasattr(error_response, "status_code"): + status_code: Final[object] = getattr(error_response, "status_code", None) + if status_code == 404: return None - elif response.status_code == 403: + elif status_code == 403: raise Exception( f"Access denied to prompt version '{prompt_version_id}'. Check your Arize Phoenix permissions." ) - elif response.status_code == 401: + elif status_code == 401: raise Exception("Authentication failed. Check your Arize Phoenix API key and permissions.") else: raise Exception(f"Failed to fetch prompt version '{prompt_version_id}': {e}") diff --git a/litellm/integrations/bitbucket/bitbucket_client.py b/litellm/integrations/bitbucket/bitbucket_client.py index e7256da0237..e4f5a50550e 100644 --- a/litellm/integrations/bitbucket/bitbucket_client.py +++ b/litellm/integrations/bitbucket/bitbucket_client.py @@ -47,6 +47,10 @@ def _sanitize_file_path(file_path: str) -> str: return "/".join(urllib.parse.quote(part, safe="") for part in parts) +def _content_type(headers: Mapping[str, str]) -> str: + return headers.get("content-type", "") + + class BitBucketClient: """ Client for interacting with BitBucket API to fetch .prompt files. @@ -119,7 +123,7 @@ class BitBucketClient: response.raise_for_status() # BitBucket returns file content as base64 encoded - if response.headers.get("content-type", "").startswith("text/"): + if _content_type(response.headers).startswith("text/"): return response.text else: # For binary files or when content-type is not text, try to decode as base64 diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 09ca4857789..2a2a2a0326d 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -145,6 +145,10 @@ def _unified_hook_fields(guardrail: "CustomGuardrail", request_data: Mapping[str } +def _instance_attributes(value: object) -> Mapping[str, object]: + return vars(value) + + def _strict_guardrail_modes_enabled() -> bool: """Whether guardrail-mode validation raises (default) or logs a warning. @@ -396,7 +400,7 @@ class CustomGuardrail(CustomLogger): def inject_advisory_message( self, - data: dict[str, Any], # mutable-ok: caller's dict is mutated in place, matching mark_pre_call_hook_ran + data: dict[str, object], # mutable-ok: caller's dict is mutated in place, matching mark_pre_call_hook_ran message: str, ) -> bool: """ @@ -473,8 +477,8 @@ class CustomGuardrail(CustomLogger): def raise_sensitive_data_route_exception( self, route_to_model: str, - request_data: dict[str, Any], - detection_info: dict[str, Any] | None = None, + request_data: dict[str, object], + detection_info: dict[str, object] | None = None, ) -> None: """ Raise an exception to reroute the request to a different model. @@ -620,8 +624,8 @@ class CustomGuardrail(CustomLogger): def handle_sensitive_data_detection( self, - request_data: dict[str, Any], - detection_info: dict[str, Any] | None = None, + request_data: dict[str, object], + detection_info: dict[str, object] | None = None, ) -> None: """ Handle sensitive data detection based on guardrail configuration. @@ -888,7 +892,9 @@ class CustomGuardrail(CustomLogger): ) from e return unified_guardrail - async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None: + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict | None: # should run guardrail litellm_guardrails: Final = kwargs.get("guardrails") if litellm_guardrails is None or not isinstance(litellm_guardrails, list): @@ -1579,7 +1585,7 @@ class CustomGuardrail(CustomLogger): """ Update the guardrails litellm params in memory """ - for key, value in vars(litellm_params).items(): + for key, value in _instance_attributes(litellm_params).items(): setattr(self, key, value) def get_guardrails_messages_for_call_type( diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 52577d0c647..5addc994df2 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -256,7 +256,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> PreRoutingHookResponse | None: diff --git a/litellm/integrations/datadog/datadog_cost_management.py b/litellm/integrations/datadog/datadog_cost_management.py index 538dd95abdd..1a81fa019dc 100644 --- a/litellm/integrations/datadog/datadog_cost_management.py +++ b/litellm/integrations/datadog/datadog_cost_management.py @@ -3,7 +3,7 @@ import os import time from collections.abc import Mapping from datetime import datetime -from typing import Any, Final, cast +from typing import Final from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger @@ -45,6 +45,10 @@ _RESERVED_TAG_KEYS: Final[frozenset] = frozenset( ) +def _as_mapping(metadata: Mapping[str, object]) -> Mapping[str, object]: + return metadata + + class DatadogCostManagementLogger(CustomBatchLogger): def __init__(self, cost_tag_keys: list[str] | None = None, **kwargs): self.cost_tag_keys: list[str] = list(cost_tag_keys) if cost_tag_keys else [] @@ -180,9 +184,7 @@ class DatadogCostManagementLogger(CustomBatchLogger): self._add_tag(tags, "model", log.get("model")) self._add_tag(tags, "model_id", log.get("model_id")) - # cast because StandardLoggingMetadata is a TypedDict; we iterate it - # as a generic mapping below. - metadata: Final[Mapping[str, object]] = cast(dict[str, Any], log.get("metadata") or {}) + metadata: Final = _as_mapping(log.get("metadata") or {}) # Backwards-compat: team/user/model_group preserved regardless of allowlist. if metadata.get("user_api_key_alias"): diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 6793838cea0..270d799742c 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -62,6 +62,7 @@ _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""} _MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024 _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_RESPONSE: Final = TypeAdapter(object) _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset( {"agent", "assistant", "developer", "function", "model", "system", "tool", "user"} ) @@ -815,11 +816,11 @@ class DataDogLLMObsLogger(CustomBatchLogger): try: import ast - response_obj = ast.literal_eval(response_obj) + response_obj = _DECODED_RESPONSE.validate_python(ast.literal_eval(response_obj)) except (ValueError, SyntaxError): try: # fallback to json parsing - response_obj = json.loads(str(response_obj)) + response_obj = _DECODED_RESPONSE.validate_python(json.loads(str(response_obj))) except json.JSONDecodeError: return () diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_base.py b/litellm/integrations/gcs_bucket/gcs_bucket_base.py index 0856a420d6f..63c4fe92eca 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_base.py @@ -2,6 +2,8 @@ import json import os from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.gcs_bucket.gcs_bucket_mock_client import ( @@ -25,6 +27,7 @@ if TYPE_CHECKING: else: VertexBase = Any IAM_AUTH_KEY: Final = "IAM_AUTH" +_UPLOAD_RESPONSE: Final = TypeAdapter(object) class GCSBucketBase(CustomBatchLogger): @@ -297,7 +300,7 @@ class GCSBucketBase(CustomBatchLogger): bucket_name: str, object_name: str, logging_payload: StandardLoggingPayload | str, - ): + ) -> object: """ Helper function to make POST request to GCS Bucket in the specified bucket. """ @@ -325,4 +328,4 @@ class GCSBucketBase(CustomBatchLogger): verbose_logger.debug("GCS Bucket status code %s", response.status_code) verbose_logger.debug("GCS Bucket response.text %s", response.text) - return response.json() + return _UPLOAD_RESPONSE.validate_python(response.json()) diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index 47a472db420..5f9851f595f 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -252,7 +252,7 @@ class GenericAPILogger(CustomBatchLogger): if self.retry_delay <= 0: return - delay: Final = self.retry_delay * (2**attempt) + delay: Final = self.retry_delay * (2.0**attempt) await asyncio.sleep(delay) async def _post_with_retries(self, data: str) -> httpx.Response: diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index 113dea541fb..85b91821f6c 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -297,7 +297,7 @@ class GitLabPromptManager(CustomPromptManagement): def __init__( self, - gitlab_config: dict[str, Any], + gitlab_config: dict[str, object], prompt_id: str | None = None, ref: str | None = None, # tag/branch/SHA override gitlab_client: GitLabClient | None = None, @@ -662,7 +662,7 @@ class GitLabPromptCache: def __init__( self, - gitlab_config: dict[str, Any], + gitlab_config: dict[str, object], *, ref: str | None = None, gitlab_client: GitLabClient | None = None, diff --git a/litellm/integrations/lago.py b/litellm/integrations/lago.py index 594427b1e0a..ec4388bbb8d 100644 --- a/litellm/integrations/lago.py +++ b/litellm/integrations/lago.py @@ -147,9 +147,10 @@ class LagoLogger(CustomLogger): response.raise_for_status() except Exception as e: - error_response: Final = getattr(e, "response", None) + error_response: Final[object] = getattr(e, "response", None) if error_response is not None and hasattr(error_response, "text"): - verbose_logger.debug("\nError Message: %s", error_response.text) + error_text: Final[object] = getattr(error_response, "text", None) + verbose_logger.debug("\nError Message: %s", error_text) raise e async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index a96fac32c2a..ae63ab99929 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -1,6 +1,7 @@ import base64 import json import os +from collections.abc import Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Optional @@ -19,7 +20,7 @@ from litellm.types.utils import StandardCallbackDynamicParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any @@ -190,18 +191,19 @@ class LangfuseOtelLogger(OpenTelemetry): } ) elif item_type == "message": + item_content: Sequence[object] = getattr(item, "content", [{}]) output_items_data.append( { "role": getattr(item, "role", "assistant"), - "content": getattr(getattr(item, "content", [{}])[0], "text", ""), + "content": getattr(item_content[0], "text", ""), } ) elif item_type == "function_call": - arguments_str = getattr(item, "arguments", "{}") + item_arguments: object = getattr(item, "arguments", "{}") arguments_obj = ( - safe_json_loads(arguments_str, default={}) - if isinstance(arguments_str, str) - else arguments_str + safe_json_loads(item_arguments, default={}) + if isinstance(item_arguments, str) + else item_arguments ) langfuse_tool_call = { "id": getattr(item, "id", ""), diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 991b0411ee8..01812149498 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -10,7 +10,7 @@ from datetime import datetime, timezone from typing import Any, Final import httpx -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._logging import verbose_logger @@ -28,6 +28,8 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.types.integrations.langsmith import * from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload +_RUN_RESPONSE: Final = TypeAdapter(object) + def is_serializable(value): non_serializable_types: Final = ( @@ -524,7 +526,7 @@ class LangsmithLogger(CustomBatchLogger): # If we can't get an event loop, create a new one asyncio.run(self.async_send_batch()) - def get_run_by_id(self, run_id): + def get_run_by_id(self, run_id) -> object: langsmith_api_key: Final = self.default_credentials["LANGSMITH_API_KEY"] langsmith_api_base: Final = self.default_credentials["LANGSMITH_BASE_URL"] langsmith_tenant_id: Final = self.default_credentials.get("LANGSMITH_TENANT_ID") @@ -538,7 +540,7 @@ class LangsmithLogger(CustomBatchLogger): headers=headers, ) - return response.json() + return _RUN_RESPONSE.validate_python(response.json()) def make_dot_order(self, run_id: str): st: Final = datetime.now(timezone.utc) diff --git a/litellm/integrations/literal_ai.py b/litellm/integrations/literal_ai.py index 65a22ff6d6d..4fab5312308 100644 --- a/litellm/integrations/literal_ai.py +++ b/litellm/integrations/literal_ai.py @@ -2,6 +2,7 @@ # This file contains the LiteralAILogger class which is used to log steps to the LiteralAI observability platform. import asyncio import os +from collections.abc import Mapping from typing import Final import httpx @@ -181,7 +182,8 @@ class LiteralAILogger(CustomBatchLogger): if messages and isinstance(messages, list) and isinstance(messages[0], dict): for message in messages: - if literal_prompt := getattr(message, "__literal_prompt__", None): + literal_prompt: Mapping[str, object] | None = getattr(message, "__literal_prompt__", None) + if literal_prompt: prompt_id = literal_prompt.get("prompt_id") variables = literal_prompt.get("variables") message["uuid"] = literal_prompt.get("uuid") diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index bf5f3d1cd24..6c020c78c32 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -3,10 +3,15 @@ import importlib import traceback from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Final import packaging +if TYPE_CHECKING: + from openai.types.chat.chat_completion_message_custom_tool_call import Custom + + from litellm.types.utils import ChatCompletionCustomToolCallPayload, ChatCompletionDeltaCustomToolCallPayload + # convert to {completion: xx, tokens: xx} def parse_usage(usage): @@ -21,7 +26,9 @@ def parse_tool_calls(tool_calls): return None def clean_tool_call(tool_call): - custom: Final = getattr(tool_call, "custom", None) + custom: Final[ + ChatCompletionCustomToolCallPayload | ChatCompletionDeltaCustomToolCallPayload | Custom | None + ] = getattr(tool_call, "custom", None) if custom is not None: name, arguments = custom.name, custom.input else: diff --git a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py index f9e54c30e89..c107279a211 100644 --- a/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py +++ b/litellm/integrations/mavvrik_focus/mavvrik_focus_logger.py @@ -24,6 +24,8 @@ from collections.abc import Callable from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final +from typing_extensions import TypedDict, Unpack + import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import MAVVRIK_FOCUS_EXPORT_JOB_NAME @@ -82,10 +84,16 @@ def _is_empty_metrics_marker(marker: object | None) -> bool: return False +class _CustomLoggerOptions(TypedDict, total=False, extra_items=object): + pass + + class MavvrikFocusLogger(FocusLogger): """FOCUS-based export logger that routes to the Mavvrik destination.""" - def __init__(self, *, clock: Callable[[], datetime] = get_utc_datetime, **kwargs: Any) -> None: + def __init__( + self, *, clock: Callable[[], datetime] = get_utc_datetime, **kwargs: Unpack[_CustomLoggerOptions] + ) -> None: self._clock: Final = clock frequency: Final = os.getenv("MAVVRIK_FOCUS_FREQUENCY", "daily").lower() if frequency != "daily": diff --git a/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py b/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py index 06cdf413e76..035e5c5db10 100644 --- a/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py +++ b/litellm/integrations/opentelemetry_utils/gen_ai_semconv.py @@ -40,6 +40,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps if TYPE_CHECKING: from opentelemetry._logs import SeverityNumber as _SeverityNumber from opentelemetry.trace import Span as _Span + from opentelemetry.trace import SpanContext from litellm.integrations.opentelemetry import OpenTelemetryConfig @@ -231,7 +232,7 @@ class OTELGenAISemconvMixin: response_obj: dict, provider: str | None, otel_logger, - parent_ctx, + parent_ctx: "SpanContext", ) -> None: """Emit the consolidated ``gen_ai.client.inference.operation.details`` log event, correlated to the request span via ``parent_ctx``. diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 4f0a30cb494..9eb9cb40caf 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -976,12 +976,16 @@ def _excluded_db_systems(logger: "OpenTelemetryV2") -> frozenset[str]: callback folds into the preset, whose config is env-only. """ otel_settings: Final = (litellm.callback_settings or {}).get("otel") - configured: Final = otel_settings.get("excluded_services") if isinstance(otel_settings, dict) else None + configured: Final = _excluded_services_setting(otel_settings) if isinstance(otel_settings, dict) else None if configured is None: return logger.config.excluded_services return excluded_db_systems_from(configured) +def _excluded_services_setting(otel_settings: Mapping[str, object]) -> object: + return otel_settings.get("excluded_services") + + def _v2_configs(in_memory_loggers: Sequence[object], logger: "OpenTelemetryV2") -> tuple[OpenTelemetryV2Config, ...]: """Every v2 logger's config, the published logger's first. diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 8524df71b44..cbbb1a9f6d4 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -42,6 +42,8 @@ from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast, get_args +from pydantic import TypeAdapter + from litellm.constants import ( INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, @@ -56,6 +58,7 @@ if TYPE_CHECKING: from litellm.types.utils import StandardLoggingPayload _INTERNAL_CALL_ORIGINS: Final[frozenset[str]] = frozenset(get_args(InternalCallOrigin)) +_AUTH_ATTRIBUTE: Final = TypeAdapter(object) REQUESTER_METADATA_KEY: Final = "requester_metadata" REQUESTER_METADATA_PATH: Final = f"{REQUESTER_METADATA_KEY}." @@ -129,7 +132,7 @@ class RequestIdentity: snapshot) is flattened to dotted keys so ``requester_metadata.`` resolves too. """ - get: Final[Callable[[str], object]] = lambda name: getattr(auth, name, None) # noqa: E731 + get: Final[Callable[[str], object]] = lambda name: _AUTH_ATTRIBUTE.validate_python(getattr(auth, name, None)) # noqa: E731 auth_meta: Final = tuple( (meta_key, str(value)) for meta_key, attr in ( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 708a75aad99..fa5496004e3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -3098,7 +3098,7 @@ class PrometheusLogger(CustomLogger): if exception is not None: exception_status = str(getattr(exception, "status_code", None)) if exception_status == "None" or not exception_status: - code: Final = getattr(exception, "code", None) + code: Final[object] = getattr(exception, "code", None) if code is not None: exception_status = str(code) diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 962c7707409..d0ed30d40dd 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -20,9 +20,10 @@ from datetime import datetime, timezone from itertools import groupby from operator import itemgetter from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, Protocol from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator, model_validator +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache @@ -801,6 +802,52 @@ def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_gro return not job.models or any(_canonical_group(router, name) == model_group for name in job.models) +class _ShadowEvalJobRow(Protocol): + @property + def id(self) -> str: ... + + @property + def target_type(self) -> str: ... + + @property + def target_id(self) -> str: ... + + +class _AttemptCount(TypedDict): + _all: ReadOnly[int] + + +class _AttemptGroup(TypedDict): + job_id: ReadOnly[str] + _count: ReadOnly[_AttemptCount] + _sum: ReadOnly[Mapping[str, object] | None] + + +class _ShadowEvalJobTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... + + +class _ShadowEvalAttemptTable(Protocol): + async def group_by( + self, + *, + by: Sequence[str], + count: bool, + sum: Mapping[str, bool], + where: Mapping[str, object], + ) -> Sequence[_AttemptGroup]: ... + + async def create(self, *, data: Mapping[str, object]) -> object: ... + + +def _shadow_eval_job_table(prisma: "PrismaClient") -> _ShadowEvalJobTable: + return prisma.db.litellm_shadowevaljob + + +def _shadow_eval_attempt_table(prisma: "PrismaClient") -> _ShadowEvalAttemptTable: + return prisma.db.litellm_shadowevalattempt + + def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None: """The sampling path's view of one job row, or None for a row it cannot sample: an unknown direction, or a reverse job with no baseline model to duplicate against. @@ -860,14 +907,14 @@ class ShadowEvalLogger(CustomLogger): if prisma is None: return _EMPTY_JOBS try: - records: Final = await prisma.db.litellm_shadowevaljob.find_many( + records: Final = await _shadow_eval_job_table(prisma).find_many( where={ "stopped_at": None, "ends_at": {"gt": datetime.now(timezone.utc)}, }, ) grouped: Final = ( - await prisma.db.litellm_shadowevalattempt.group_by( + await _shadow_eval_attempt_table(prisma).group_by( by=["job_id"], count=True, sum={"judge_cost": True, "shadow_cost": True, "shadow_classifier_cost": True}, @@ -894,7 +941,10 @@ class ShadowEvalLogger(CustomLogger): ) ) jobs: Final = MappingProxyType( - {target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))} + { + target: tuple(job for _, job in group) + for target, group in groupby(by_target, key=lambda entry: entry[0]) + } ) await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs) self._job_starts = {} @@ -1242,7 +1292,7 @@ class ShadowEvalLogger(CustomLogger): if prisma is None: return try: - await prisma.db.litellm_shadowevalattempt.create( + await _shadow_eval_attempt_table(prisma).create( data={ "job_id": job.id, "request_id": request_id, diff --git a/litellm/integrations/vantage/vantage_logger.py b/litellm/integrations/vantage/vantage_logger.py index 79beac78fcc..a6389310e19 100644 --- a/litellm/integrations/vantage/vantage_logger.py +++ b/litellm/integrations/vantage/vantage_logger.py @@ -9,6 +9,8 @@ from __future__ import annotations import os from typing import TYPE_CHECKING, Any, Final, cast +from typing_extensions import TypedDict, Unpack + import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -24,6 +26,10 @@ else: VANTAGE_USAGE_DATA_JOB_NAME: Final = "vantage_export_usage_data" +class _CustomLoggerOptions(TypedDict, total=False, extra_items=object): + pass + + class VantageLogger(FocusLogger): """FocusLogger pre-configured for Vantage (CSV format, Vantage API destination). @@ -43,7 +49,7 @@ class VantageLogger(FocusLogger): base_url: str | None = None, frequency: str | None = None, interval_seconds: int | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomLoggerOptions], ) -> None: resolved_api_key: Final = api_key or os.getenv("VANTAGE_API_KEY") resolved_token: Final = integration_token or os.getenv("VANTAGE_INTEGRATION_TOKEN") diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 47af73570fc..bbb1b4d494b 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -6,8 +6,9 @@ Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format. import json from collections.abc import Sequence -from typing import Any, Final +from typing import Final +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -21,6 +22,8 @@ from litellm.types.integrations.websearch_interception import ( WebSearchToolResultErrorCode, ) +_DECODED_JSON: Final = TypeAdapter(object) + class WebSearchTransformation: """ @@ -138,7 +141,7 @@ class WebSearchTransformation: @staticmethod def _detect_from_non_streaming_response( - response: Any, + response: object, ) -> tuple[bool, list[dict]]: """Parse non-streaming response for WebSearch tool_use""" @@ -149,7 +152,7 @@ class WebSearchTransformation: if not hasattr(response, "content"): verbose_logger.debug("WebSearchInterception: Response has no content attribute") return False, [] - content = response.content or [] + content = getattr(response, "content", None) or [] if not content: verbose_logger.debug("WebSearchInterception: Response has empty content") @@ -194,7 +197,7 @@ class WebSearchTransformation: @staticmethod def _detect_from_openai_response( - response: Any, + response: object, ) -> tuple[bool, list[dict]]: """Parse OpenAI-style response for WebSearch tool_calls""" @@ -205,7 +208,7 @@ class WebSearchTransformation: if not hasattr(response, "choices"): verbose_logger.debug("WebSearchInterception: Response has no choices attribute") return False, [] - choices = response.choices or [] + choices = getattr(response, "choices", None) or [] if not choices: verbose_logger.debug("WebSearchInterception: Response has empty choices") @@ -247,9 +250,9 @@ class WebSearchTransformation: else: tool_id = getattr(tool_call, "id", None) tool_type = getattr(tool_call, "type", None) - function = getattr(tool_call, "function", None) - function_name = getattr(function, "name", None) if function else None - function_arguments = getattr(function, "arguments", None) if function else None + function_obj: object = getattr(tool_call, "function", None) + function_name = getattr(function_obj, "name", None) if function_obj else None + function_arguments = getattr(function_obj, "arguments", None) if function_obj else None # Detect function-style web search tool_calls. ``WebSearch`` is # intentionally omitted — see is_web_search_tool for the Cowork @@ -262,7 +265,7 @@ class WebSearchTransformation: # Parse arguments (might be JSON string) if isinstance(function_arguments, str): try: - arguments = json.loads(function_arguments) + arguments = _DECODED_JSON.validate_python(json.loads(function_arguments)) except json.JSONDecodeError: verbose_logger.warning( "WebSearchInterception: Failed to parse function arguments: %s", function_arguments diff --git a/litellm/interactions/agents/main.py b/litellm/interactions/agents/main.py index 1ca28adf0a4..02b0ef4807f 100644 --- a/litellm/interactions/agents/main.py +++ b/litellm/interactions/agents/main.py @@ -101,7 +101,7 @@ async def acreate( instructions: str | None = None, base_environment: InteractionEnvironment | None = None, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, @@ -145,7 +145,7 @@ def create( instructions: str | None = None, base_environment: InteractionEnvironment | None = None, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, @@ -207,7 +207,7 @@ def create( @client async def alist( custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentListResponse: @@ -241,7 +241,7 @@ async def alist( @client def list( custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentListResponse | Coroutine[object, object, AgentListResponse]: @@ -281,7 +281,7 @@ def list( async def aget( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentCreateResponse: @@ -317,7 +317,7 @@ async def aget( def get( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]: @@ -358,7 +358,7 @@ def get( async def adelete( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentDeleteResult: @@ -394,7 +394,7 @@ async def adelete( def delete( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]: @@ -435,7 +435,7 @@ def delete( async def alist_versions( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentVersionsResponse: @@ -471,7 +471,7 @@ async def alist_versions( def list_versions( name: str, custom_llm_provider: str | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, **kwargs, ) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]: diff --git a/litellm/interactions/http_handler.py b/litellm/interactions/http_handler.py index f02e7e7679d..09aa656e49b 100644 --- a/litellm/interactions/http_handler.py +++ b/litellm/interactions/http_handler.py @@ -4,8 +4,8 @@ HTTP Handler for Interactions API requests. This module handles the HTTP communication for the Google Interactions API. """ -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping -from typing import Any, Final +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping +from typing import Final, Protocol import httpx @@ -34,6 +34,11 @@ from litellm.types.interactions import ( from litellm.types.router import GenericLiteLLMParams +class _ErrorClassSource(Protocol): + @property + def get_error_class(self) -> Callable[..., Exception]: ... + + class _BaseHTTPHandler: """ Shared HTTP infrastructure for LiteLLM handler classes. @@ -43,7 +48,7 @@ class _BaseHTTPHandler: not duplicate this boilerplate. """ - def _handle_error(self, e: Exception, provider_config: Any) -> Exception: + def _handle_error(self, e: Exception, provider_config: _ErrorClassSource) -> Exception: if isinstance(e, httpx.HTTPStatusError): return provider_config.get_error_class( error_message=e.response.text, @@ -77,7 +82,7 @@ class InteractionsHTTPHandler(_BaseHTTPHandler): HTTP handler for Interactions API requests. """ - # _handle_error is inherited from _BaseHTTPHandler (accepts Any provider_config). + # _handle_error is inherited from _BaseHTTPHandler (accepts any config exposing get_error_class). # AgentsHTTPHandler also extends this class and passes BaseAgentsAPIConfig, which # is structurally compatible but a different type — keeping the override here with # BaseInteractionsAPIConfig would cause type errors in the subclass. diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index 74c1799190d..498e6e20aec 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -33,9 +33,9 @@ Usage: import asyncio import contextvars -from collections.abc import AsyncIterator, Coroutine, Iterator +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping from functools import partial -from typing import Any, Final +from typing import Final import httpx @@ -79,7 +79,7 @@ async def acreate( # System instruction system_instruction: str | None = None, # Generation config - generation_config: dict[str, Any] | None = None, + generation_config: Mapping[str, object] | None = None, # Streaming stream: bool | None = None, # Storage @@ -90,13 +90,13 @@ async def acreate( environment: InteractionEnvironment | None = None, # Response format response_modalities: list[str] | None = None, - response_format: dict[str, Any] | None = None, + response_format: Mapping[str, object] | None = None, response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, # Extra params - extra_headers: dict[str, Any] | None = None, - extra_body: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, + extra_body: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, # LiteLLM params custom_llm_provider: str | None = None, @@ -204,7 +204,7 @@ def create( # System instruction system_instruction: str | None = None, # Generation config - generation_config: dict[str, Any] | None = None, + generation_config: Mapping[str, object] | None = None, # Streaming stream: bool | None = None, # Storage @@ -215,13 +215,13 @@ def create( environment: InteractionEnvironment | None = None, # Response format response_modalities: list[str] | None = None, - response_format: dict[str, Any] | None = None, + response_format: Mapping[str, object] | None = None, response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, # Extra params - extra_headers: dict[str, Any] | None = None, - extra_body: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, + extra_body: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, # LiteLLM params custom_llm_provider: str | None = None, @@ -359,7 +359,7 @@ def create( @client async def aget( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -402,7 +402,7 @@ async def aget( @client def get( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -461,7 +461,7 @@ def get( @client async def adelete( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs: object, @@ -506,7 +506,7 @@ async def adelete( @client def delete( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -565,7 +565,7 @@ def delete( @client async def acancel( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -608,7 +608,7 @@ async def acancel( @client def cancel( interaction_id: str, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, str] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index c12eb4d458f..ff7a232e83a 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -3,9 +3,9 @@ import copy import logging import re -from collections.abc import Collection, Iterable, Mapping +from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable import httpx from pydantic import TypeAdapter, ValidationError @@ -56,6 +56,33 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None +@runtime_checkable +class _SupportsIter(Protocol): + def __iter__(self) -> Iterator[object]: ... + + +@runtime_checkable +class _SupportsGetItem(Protocol): + def __getitem__(self, index: int, /) -> object: ... + + +def attribute_or_none(source: object, name: str) -> object: + return getattr(source, name, None) + + +def iter_items(value: object) -> Iterable[object]: + if isinstance(value, _SupportsIter): + return value + if isinstance(value, _SupportsGetItem): + return iter(value) + raise TypeError(f"'{type(value).__name__}' object is not iterable") + + +def attribute_items(source: object, name: str) -> Iterable[object]: + value: Final = attribute_or_none(source, name) + return iter_items(value) if value else () + + def normalize_drop_params(value: object) -> bool | None: if value is None or isinstance(value, bool): return value @@ -275,7 +302,13 @@ def remove_index_from_tool_calls( tool_call.pop("index", None) -def remove_items_at_indices(items: list[Any] | None, indices: Iterable[int]) -> None: +class _PoppableByIndex(Protocol): + def __len__(self) -> int: ... + + def pop(self, index: int, /) -> object: ... + + +def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[int]) -> None: """Remove items from a list in-place by index""" if items is None: return diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 82d3ea0adb9..f8aadc4258d 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.bug_report import ( build_bug_report, should_report_bug, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string from litellm.types.utils import LlmProviders @@ -185,12 +186,20 @@ def get_error_message(error_obj) -> str | None: ####### EXCEPTION MAPPING ################ +_DECODED_ERROR_BODY: Final = TypeAdapter(object) + + def _get_body_error_code(error_str: str) -> int | None: """Return error.code from a JSON error body, or None if not parseable.""" try: - body: Final = json.loads(error_str) - code: Final = body.get("error", {}).get("code") - return int(code) if code is not None else None + body: Final = as_str_mapping(_DECODED_ERROR_BODY.validate_python(json.loads(error_str))) + if body is None: + return None + error: Final = as_str_mapping(body.get("error", {})) + if error is None: + return None + code: Final = error.get("code") + return int(code) if isinstance(code, (str, int, float)) else None except Exception: return None diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 52fb690a610..fee48cca0cf 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -241,8 +241,9 @@ class _FetchAttemptRetryable: def _parse_retry_after_seconds(response: httpx.Response) -> float | None: - header: Final[str | None] = response.headers.get("Retry-After") - if header is None: + try: + header: Final = response.headers["Retry-After"] + except KeyError: return None try: seconds: Final = float(header) @@ -254,7 +255,7 @@ def _parse_retry_after_seconds(response: httpx.Response) -> float | None: def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float: if outcome.retry_after_seconds is not None: return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) - return min(float(2**attempt) + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) + return min(2.0**attempt + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) class _AsyncGetClient(Protocol): diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 77d3b255fd7..49845bd07b5 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -282,7 +282,11 @@ def get_supported_openai_params( return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "soniox": if request_type == "transcription": - return litellm.SonioxAudioTranscriptionConfig().get_supported_openai_params(model=model) + from litellm.llms.soniox.audio_transcription.transformation import ( + SonioxAudioTranscriptionConfig, + ) + + return SonioxAudioTranscriptionConfig().get_supported_openai_params(model=model) elif custom_llm_provider in litellm._custom_providers: if request_type == "chat_completion": provider_config = litellm.ProviderConfigManager.get_provider_chat_config( diff --git a/litellm/litellm_core_utils/json_validation_rule.py b/litellm/litellm_core_utils/json_validation_rule.py index 9fd4c03ac9e..b6b16aff7a7 100644 --- a/litellm/litellm_core_utils/json_validation_rule.py +++ b/litellm/litellm_core_utils/json_validation_rule.py @@ -1,8 +1,12 @@ import json from typing import Any, Final +from pydantic import TypeAdapter + from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +_DECODED_RESPONSE: Final = TypeAdapter(object) + def normalize_json_schema_types( schema: object, @@ -109,7 +113,7 @@ def validate_schema(schema: dict, response: str): from litellm import JSONSchemaValidationError try: - response_dict: Final = json.loads(response) + response_dict: Final = _DECODED_RESPONSE.validate_python(json.loads(response)) except json.JSONDecodeError: raise JSONSchemaValidationError(model="", llm_provider="", raw_response=response, schema=json.dumps(schema)) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index bbeacc3c584..0cda24c853b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -609,6 +609,10 @@ class Logging(LiteLLMLoggingBaseClass): def _on_detached_stream_failure(self, value: Callable[[Exception], Awaitable[None]] | None) -> None: self.on_detached_stream_failure = value + @property + def _model_call_details_view(self) -> Mapping[str, object]: + return self.model_call_details + def __init__( self, model: str, @@ -1467,13 +1471,13 @@ class Logging(LiteLLMLoggingBaseClass): try: if callback == "supabase" and supabaseClient is not None: verbose_logger.debug("reaches supabase for logging!") - model = self.model_call_details["model"] - messages = self.model_call_details["input"] + model = self._model_call_details_view["model"] + messages = self._model_call_details_view["input"] verbose_logger.debug("supabaseClient: %s", supabaseClient) supabaseClient.input_log_event( model=model, messages=messages, - end_user=self.model_call_details.get("user", "default"), + end_user=self._model_call_details_view.get("user", "default"), litellm_call_id=self.litellm_params["litellm_call_id"], print_verbose=print_verbose, ) @@ -1617,7 +1621,7 @@ class Logging(LiteLLMLoggingBaseClass): callattr: Final = verbose_logger.warning if attr == "warning" else verbose_logger.debug callattr( "RAW RESPONSE:\n{}\n\n".format( - self.model_call_details.get("original_response", self.model_call_details) + self._model_call_details_view.get("original_response", self._model_call_details_view) ) ) if getattr(self, "logger_fn", None) and callable(self.logger_fn): @@ -1755,8 +1759,8 @@ class Logging(LiteLLMLoggingBaseClass): def get_response_ms(self) -> float: now: Final = datetime.datetime.now() - start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now)) - end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now)) + start_seconds: Final = _timestamp_seconds(self._model_call_details_view.get("start_time", now)) + end_seconds: Final = _timestamp_seconds(self._model_call_details_view.get("end_time", now)) if start_seconds is None or end_seconds is None: return 0.0 return (end_seconds - start_seconds) * 1000 @@ -2522,6 +2526,11 @@ class Logging(LiteLLMLoggingBaseClass): self.callback_duration_ms += (time.time() - _start) * 1000 return payload + def _emit_stored_standard_logging_payload(self) -> None: + stored_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get("standard_logging_object") + if stored_payload is not None: + emit_standard_logging_payload(stored_payload) + def _transform_usage_objects(self, result: object) -> object: if isinstance(result, ResponsesAPIResponse): result = result.model_copy() @@ -2956,10 +2965,10 @@ class Logging(LiteLLMLoggingBaseClass): continue else: print_verbose("reaches supabase for streaming logging!") - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] - model = kwargs["model"] - messages = kwargs["messages"] + model = self._model_call_details_view["model"] + messages = self._model_call_details_view["messages"] optional_params = kwargs.get("optional_params", {}) litellm_params = kwargs.get("litellm_params", {}) supabaseClient.log_event( @@ -2988,7 +2997,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback == "logfire" and logfireLogger is not None: verbose_logger.debug("reaches logfire for success logging!") kwargs = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine kwargs[k] = v @@ -2998,7 +3007,7 @@ class Logging(LiteLLMLoggingBaseClass): continue else: print_verbose("reaches logfire for streaming logging!") - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] logfireLogger.log_event( kwargs=self.model_call_details, @@ -3023,7 +3032,7 @@ class Logging(LiteLLMLoggingBaseClass): if "complete_streaming_response" not in kwargs: continue else: - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] lunaryLogger.log_event( type=type, @@ -3033,7 +3042,7 @@ class Logging(LiteLLMLoggingBaseClass): input=input, user_id=kwargs.get("user", None), # user_props=self.model_call_details.get("user_props", None), - extra=kwargs.get("optional_params", {}), + extra=self._model_call_details_view.get("optional_params", {}), response_obj=result, start_time=start_time, end_time=end_time, @@ -3043,7 +3052,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback == "helicone" and heliconeLogger is not None: print_verbose("reaches helicone for logging!") model = self.model - messages = self.model_call_details["input"] + messages = self._model_call_details_view["input"] kwargs = self.model_call_details # this only logs streaming once, complete_streaming_response exists i.e when stream ends @@ -3052,7 +3061,7 @@ class Logging(LiteLLMLoggingBaseClass): continue else: print_verbose("reaches helicone for streaming logging!") - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] heliconeLogger.log_success( model=model, @@ -3067,7 +3076,7 @@ class Logging(LiteLLMLoggingBaseClass): global langFuseLogger print_verbose("reaches langfuse for success logging!") kwargs = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine kwargs[k] = v # this only logs streaming once, complete_streaming_response exists i.e when stream ends @@ -3080,7 +3089,7 @@ class Logging(LiteLLMLoggingBaseClass): continue else: print_verbose("reaches langfuse for streaming logging!") - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request( globalLangfuseLogger=langFuseLogger, @@ -3105,7 +3114,7 @@ class Logging(LiteLLMLoggingBaseClass): ) if callback == "greenscale" and greenscaleLogger is not None: kwargs = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine kwargs[k] = v # this only logs streaming once, complete_streaming_response exists i.e when stream ends @@ -3118,7 +3127,7 @@ class Logging(LiteLLMLoggingBaseClass): continue else: print_verbose("reaches greenscale for streaming logging!") - result = kwargs["complete_streaming_response"] + result = self._model_call_details_view["complete_streaming_response"] greenscaleLogger.log_event( kwargs=kwargs, @@ -3129,7 +3138,7 @@ class Logging(LiteLLMLoggingBaseClass): ) if callback == "athina" and athinaLogger is not None: deep_copy = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): deep_copy[k] = v athinaLogger.log_event( kwargs=deep_copy, @@ -3140,7 +3149,7 @@ class Logging(LiteLLMLoggingBaseClass): ) if callback == "traceloop": deep_copy = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": deep_copy[k] = v traceloopLogger.log_event( @@ -3193,7 +3202,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_response"] = self.model_call_details.get( "complete_streaming_response", {} ) - result = self.model_call_details["complete_response"] + result = self._model_call_details_view["complete_response"] openMeterLogger.log_success_event( kwargs=self.model_call_details, response_obj=result, @@ -3218,7 +3227,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_response"] = self.model_call_details.get( "complete_streaming_response", {} ) - result = self.model_call_details["complete_response"] + result = self._model_call_details_view["complete_response"] callback.log_success_event( kwargs=self.model_call_details, @@ -3369,8 +3378,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( result, start_time, end_time ) - if (prepared_payload := self.model_call_details.get("standard_logging_object")) is not None: - emit_standard_logging_payload(prepared_payload) + self._emit_stored_standard_logging_payload() ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: @@ -3434,8 +3442,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # print standard logging payload - if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None: - emit_standard_logging_payload(standard_logging_payload) + self._emit_stored_standard_logging_payload() elif self.call_type == "pass_through_endpoint": print_verbose("Async success callbacks: Got a pass-through endpoint response") @@ -3457,8 +3464,7 @@ class Logging(LiteLLMLoggingBaseClass): ) # print standard logging payload - if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None: - emit_standard_logging_payload(standard_logging_payload) + self._emit_stored_standard_logging_payload() callbacks: Final = self.get_combined_callback_list( dynamic_success_callbacks=self.dynamic_async_success_callbacks, global_callbacks=litellm._async_success_callback, @@ -3815,7 +3821,7 @@ class Logging(LiteLLMLoggingBaseClass): model = self.model - input = self.model_call_details["input"] + input = self._model_call_details_view["input"] _type = "embed" if self.call_type == CallTypes.embedding.value else "llm" @@ -3823,7 +3829,7 @@ class Logging(LiteLLMLoggingBaseClass): kwargs=self.model_call_details, type=_type, event="error", - user_id=self.model_call_details.get("user", "default"), + user_id=self._model_call_details_view.get("user", "default"), model=model, input=input, error=traceback_exception, @@ -3844,11 +3850,11 @@ class Logging(LiteLLMLoggingBaseClass): supabaseClient.log_event( model=self.model if hasattr(self, "model") else "", messages=self.messages, - end_user=self.model_call_details.get("user", "default"), + end_user=self._model_call_details_view.get("user", "default"), response_obj=result, start_time=start_time, end_time=end_time, - litellm_call_id=self.model_call_details["litellm_call_id"], + litellm_call_id=self._model_call_details_view["litellm_call_id"], print_verbose=print_verbose, ) if callable(callback): # custom logger functions @@ -3878,7 +3884,7 @@ class Logging(LiteLLMLoggingBaseClass): global langFuseLogger verbose_logger.debug("reaches langfuse for logging failure") kwargs = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine kwargs[k] = v # this only logs streaming once, complete_streaming_response exists i.e when stream ends @@ -3918,7 +3924,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback == "logfire" and logfireLogger is not None: verbose_logger.debug("reaches logfire for failure logging!") kwargs = {} - for k, v in self.model_call_details.items(): + for k, v in self._model_call_details_view.items(): if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine kwargs[k] = v kwargs["exception"] = exception @@ -4379,7 +4385,7 @@ class Logging(LiteLLMLoggingBaseClass): ) model_response: Final = litellm.ModelResponse(id=served_id) model_response.model = self.model - usage: Final = getattr(result, "usage", None) + usage: Final = result.usage if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage): setattr( model_response, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 63bbd6dc904..f32fba7c6ec 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -405,6 +405,7 @@ class _ParallelToolUseArgs(TypedDict): _PARALLEL_TOOL_USE_ARGS: Final = TypeAdapter(_ParallelToolUseArgs) +_DECODED_TOOL_ARGUMENTS: Final = TypeAdapter(object) class _CachedMessageProviderFields(TypedDict, total=False): @@ -453,7 +454,7 @@ def handle_invalid_parallel_tool_calls( if isinstance(tool_call, ChatCompletionMessageCustomToolCall): continue current_function = tool_call.function.name - function_args = json.loads(tool_call.function.arguments) + function_args = _DECODED_TOOL_ARGUMENTS.validate_python(json.loads(tool_call.function.arguments)) if current_function == "multi_tool_use.parallel": verbose_logger.debug("OpenAI did a weird pseudo-multi-tool-use call, fixing call structure..") for _fake_i, _fake_tool_use in enumerate( diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 0e8ca2147fc..ff82aacf393 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -204,7 +204,7 @@ class ResponseMetadata: ######################################################### # 3. Detailed per-phase timing (opt-in via env var) ######################################################### - llm_api_duration_ms: Final = logging_obj.model_call_details.get("llm_api_duration_ms") + llm_api_duration_ms: Final[float | None] = logging_obj.model_call_details.get("llm_api_duration_ms") if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None: detailed: Final[dict[str, float]] = { "timing_llm_api_ms": round(llm_api_duration_ms, 4), diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 5eaa4b467fe..9f521ffc6cc 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -29,7 +29,7 @@ class LoggingCallbackManager: # healthy maximum number of callbacks - unlikely someone needs more than 20 MAX_CALLBACKS = 30 - def _is_async_callable(self, callback) -> bool: + def _is_async_callable(self, callback: object) -> bool: """Check if a callback is async. Used to auto-route callbacks to the correct list.""" try: from litellm.litellm_core_utils.coroutine_checker import coroutine_checker diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index aeaca64693c..81fc865a5bc 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2539,6 +2539,9 @@ def merge_consecutive_system_messages( ] +_DECODED_JSON: Final = TypeAdapter(object) + + def _attempt_json_repair(s: str) -> object | None: """ Attempt to repair truncated JSON produced by LLM tool calls. @@ -2590,7 +2593,7 @@ def _attempt_json_repair(s: str) -> object | None: candidate += "".join(reversed(opener_stack)) try: - return json.loads(candidate) + return _DECODED_JSON.validate_python(json.loads(candidate)) except json.JSONDecodeError: pass @@ -2629,7 +2632,7 @@ def parse_tool_call_arguments( return {} try: - return json.loads(arguments) + return _DECODED_JSON.validate_python(json.loads(arguments)) except json.JSONDecodeError as original_error: repaired: Final = _attempt_json_repair(arguments) if repaired is not None: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 7b427d786a2..fc4a19f2a1d 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import Any, Final, TypeAlias, TypedDict, cast, overload from jinja2.sandbox import ImmutableSandboxedEnvironment -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, JsonValue, TypeAdapter import litellm import litellm.types @@ -28,7 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import parts_of, ) from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get -from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client +from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value from litellm.types.files import get_file_extension_from_mime_type from litellm.types.llms.anthropic import * from litellm.types.llms.bedrock import CachePointBlock @@ -216,6 +216,7 @@ def _handle_ollama_system_message(messages: list, prompt: str, msg_i: int) -> tu _OLLAMA_USER_ROLES: Final = frozenset({"user", "tool", "function"}) +_DECODED_JSON: Final = TypeAdapter(object) def _ollama_bad_message(model: str, message: AllMessageValues, msg_i: int, detail: str) -> "litellm.BadRequestError": @@ -1291,7 +1292,7 @@ def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None: if signature_from_function_fields: return cast(str, signature_from_function_fields) # cast-ok: untyped provider response field elif hasattr(function, "provider_specific_fields") and getattr(function, "provider_specific_fields"): - function_provider_fields: Final[object] = getattr(function, "provider_specific_fields") + function_provider_fields: Final[object] = getattr(function, "provider_specific_fields", None) if isinstance(function_provider_fields, dict): typed_function_provider_fields: Final = cast( # cast-ok: provider fields are dynamic dict[str, object], function_provider_fields @@ -3032,7 +3033,7 @@ def parse_xml_params(xml_content, json_schema: dict | None = None): for value in _element: try: if value.text is not None: - _value = json.loads(value.text) + _value = _DECODED_JSON.validate_python(json.loads(value.text)) else: continue except json.JSONDecodeError: @@ -3042,7 +3043,7 @@ def parse_xml_params(xml_content, json_schema: dict | None = None): # If property is not an array, append the value directly elif _element is not None and _element.text is not None: try: - _value = json.loads(_element.text) + _value = _DECODED_JSON.validate_python(json.loads(_element.text)) except json.JSONDecodeError: _value = _element.text params[prop] = _value @@ -3402,7 +3403,7 @@ def _load_image_from_url(image_url): response.raise_for_status() # Raise an exception for HTTP errors # Check the response's content type to ensure it is an image - content_type: Final = response.headers.get("content-type") + content_type: Final = header_value(response.headers, "content-type") if not content_type or "image" not in content_type: raise ValueError(f"URL does not point to a valid image (content-type: {content_type})") @@ -3604,7 +3605,7 @@ class BedrockImageProcessor: @staticmethod def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> tuple[str, str]: # Check the response's content type to ensure it is an image - content_type: str | None = response.headers.get("content-type") + content_type: str | None = header_value(response.headers, "content-type") # Use helper function to infer content type with fallback logic content_type = infer_content_type_from_url_and_content( diff --git a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py index 6ab65d5c09c..d2531e70d6b 100644 --- a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py +++ b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py @@ -2,6 +2,7 @@ import json from datetime import datetime from typing import Any, Final, Literal +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import ( @@ -10,6 +11,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.types.llms.custom_http import httpxSpecialProvider +_DECODED_TOKENIZER_CONFIG: Final = TypeAdapter(object) + class _TokenizerConfigResult(TypedDict): """Outcome of a tokenizer_config.json fetch, carrying the parsed document when the fetch succeeded.""" @@ -55,7 +58,7 @@ def get_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: except Exception as e: raise e if response.status_code == 200: - tokenizer_config: Final = json.loads(response.content) + tokenizer_config: Final = _DECODED_TOKENIZER_CONFIG.validate_python(json.loads(response.content)) return {"status": "success", "tokenizer": tokenizer_config} else: return {"status": "failure"} @@ -83,7 +86,7 @@ async def aget_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: except Exception as e: raise e if response.status_code == 200: - tokenizer_config: Final = json.loads(response.content) + tokenizer_config: Final = _DECODED_TOKENIZER_CONFIG.validate_python(json.loads(response.content)) return {"status": "success", "tokenizer": tokenizer_config} else: return {"status": "failure"} diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 4cf76e5bd5e..3287515c81d 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -29,6 +29,13 @@ _FETCH_ATTEMPTS: Final = 3 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY) +def _header_value(response: Response, name: str) -> str | None: + try: + return response.headers[name] + except KeyError: + return None + + def _process_image_response(response: Response, url: str) -> str: if response.status_code != 200: raise litellm.ImageFetchError( @@ -36,7 +43,7 @@ def _process_image_response(response: Response, url: str) -> str: ) # Check size before downloading if Content-Length header is present - content_length: Final = response.headers.get("Content-Length") + content_length: Final = _header_value(response, "Content-Length") if content_length is not None: size_mb = int(content_length) / (1024 * 1024) if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: @@ -64,7 +71,7 @@ def _process_image_response(response: Response, url: str) -> str: img_type: Final = infer_content_type_from_url_and_content( url=url, content=bytes(image_bytes), - current_content_type=response.headers.get("Content-Type"), + current_content_type=_header_value(response, "Content-Type"), ) except ValueError as e: raise litellm.ImageFetchError( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 0b659b31d00..c9b96c0a4c2 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from enum import Enum, auto from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly import litellm @@ -107,6 +108,9 @@ class _LoggingWorker(Protocol): def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: ... +_DECODED_JSON: Final = TypeAdapter(object) + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -293,11 +297,13 @@ class RealTimeStreaming: """Extract user text content from client WebSocket events for spend logging.""" try: if isinstance(message, str): - msg_obj = json.loads(message) + msg_obj = _DECODED_JSON.validate_python(json.loads(message)) elif isinstance(message, dict): msg_obj = message else: return + if not isinstance(msg_obj, dict): + return msg_type: Final = msg_obj.get("type", "") @@ -773,7 +779,7 @@ class RealTimeStreaming: if not self._has_audio_transcription_guardrails(): return setup_message try: - obj: Final = json.loads(setup_message) + obj: Final = _DECODED_JSON.validate_python(json.loads(setup_message)) except (json.JSONDecodeError, TypeError): return setup_message setup: Final = obj.get("setup") if isinstance(obj, dict) else None diff --git a/litellm/litellm_core_utils/safe_json_loads.py b/litellm/litellm_core_utils/safe_json_loads.py index a0ef6f6fdd9..6045b8012f0 100644 --- a/litellm/litellm_core_utils/safe_json_loads.py +++ b/litellm/litellm_core_utils/safe_json_loads.py @@ -3,7 +3,11 @@ Helper for safe JSON loading in LiteLLM. """ import json -from typing import Any +from typing import Any, Final + +from pydantic import TypeAdapter + +_DECODED_JSON: Final = TypeAdapter(object) def safe_json_loads(data: str, default: object = None) -> Any: @@ -11,6 +15,6 @@ def safe_json_loads(data: str, default: object = None) -> Any: Safely parse a JSON string. If parsing fails, return the default value (None by default). """ try: - return json.loads(data) + return _DECODED_JSON.validate_python(json.loads(data)) except Exception: return default diff --git a/litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py b/litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py index 73aca909ce3..9361fc8dec4 100644 --- a/litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py +++ b/litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py @@ -69,6 +69,6 @@ class DynamicLoggingCache: response: Final = self.cache.get_cache(key=key_name) return response - def set_cache(self, credentials: dict, service_name: str, logging_obj: Any) -> None: + def set_cache(self, credentials: dict, service_name: str, logging_obj: object) -> None: key_name: Final = self.get_cache_key(args={**credentials, "service_name": service_name}) self.cache.set_cache(key=key_name, value=logging_obj) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 484978bae04..83511456d45 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -81,6 +81,11 @@ def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming setattr(response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field +def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, object]) -> None: + for key, value in fields.items(): + setattr(response, key, value) + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -857,8 +862,7 @@ class CustomStreamWrapper: provider_specific_fields: Final = getattr(original_chunk, "provider_specific_fields", None) if provider_specific_fields is not None: model_response.provider_specific_fields = provider_specific_fields - for k, v in provider_specific_fields.items(): - setattr(model_response, k, v) + _apply_provider_fields(model_response, provider_specific_fields) return model_response def is_chunk_non_empty( @@ -1211,8 +1215,7 @@ class CustomStreamWrapper: "provider_specific_fields" in anthropic_response_obj and anthropic_response_obj["provider_specific_fields"] is not None ): - for key, value in anthropic_response_obj["provider_specific_fields"].items(): - setattr(model_response, key, value) + _apply_provider_fields(model_response, anthropic_response_obj["provider_specific_fields"]) response_obj = cast(dict[str, object], anthropic_response_obj) elif self.model == "replicate" or self.custom_llm_provider == "replicate": diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 717bafae5e7..6bc1c4ead64 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -24,6 +24,7 @@ from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, @@ -225,6 +226,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge _TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_SSE_DATA: Final = TypeAdapter(object) _RELEASED_TOOL_USE_STOP: Final = ( b"event: message_delta\n" b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, ' @@ -358,8 +360,8 @@ class AnthropicMessagesHandler(BaseTranslation): def build_stream_error_items( self, exc: "HTTPException", - responses_so_far: Sequence[Any] | None = None, - ) -> Sequence[Any] | None: + responses_so_far: Sequence[object] | None = None, + ) -> Sequence[object] | None: from litellm.proxy.common_request_processing import ( serialize_http_exception_detail, ) @@ -473,12 +475,13 @@ class AnthropicMessagesHandler(BaseTranslation): if not line.startswith("data:"): return () try: - parsed: Final[object] = json.loads(line[len("data:") :].strip()) + parsed: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip())) except json.JSONDecodeError: return () - if not isinstance(parsed, dict): + event: Final = as_str_mapping(parsed) + if event is None: return () - return (_as_str_mapping(parsed),) + return (event,) @staticmethod def _iter_sse_events(item: object) -> Sequence[Mapping[str, object]]: diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index a68ca1f5534..718536120b6 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -11,6 +11,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast import httpx +from pydantic import TypeAdapter import litellm import litellm.litellm_core_utils @@ -81,6 +82,9 @@ def _loads_stream_chunk(payload: str) -> dict[str, object]: return json.loads(payload) +_DECODED_JSON: Final = TypeAdapter(object) + + _EnvelopeVerdict = Literal["undecided", "pass_through", "hold"] _FIRST_OBJECT_KEY: Final = re.compile(r'^\s*\{\s*"((?:[^"\\]|\\.)*)"\s*:') _OBJECT_KEY_PREFIX: Final = re.compile(r'^\s*(?:\{\s*(?:"(?:[^"\\]|\\.)*\\?(?:"\s*)?)?)?$') @@ -164,7 +168,7 @@ def _prefix_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: def _held_answer(held: str, schema: ResponseFormatSchema) -> str: try: - payload: Final[object] = json.loads(held) + payload: Final = _DECODED_JSON.validate_python(json.loads(held)) except (json.JSONDecodeError, RecursionError): return held contents: Final = response_format_envelope_contents(payload, schema, RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index a8e49870483..f9cb1cbf091 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -523,7 +523,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) # Include caller information if present (for programmatic tool calling) if "caller" in anthropic_tool_content: - tool_call["caller"] = cast(dict[str, Any], anthropic_tool_content["caller"]) + tool_call["caller"] = cast(dict[str, object], anthropic_tool_content["caller"]) return tool_call @staticmethod @@ -1707,8 +1707,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def map_openai_context_management_to_anthropic( - context_management: list[dict[str, Any]] | dict[str, Any], - ) -> dict[str, Any] | None: + context_management: Sequence[Mapping[str, object]] | dict[str, object], + ) -> dict[str, object] | None: """ OpenAI format: [{"type": "compaction", "compact_threshold": 200000}] Anthropic format: { @@ -1732,7 +1732,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # If in OpenAI format (list), transform to Anthropic format if isinstance(context_management, list): - anthropic_edits: Final = [] + anthropic_edits: Final[list[dict[str, object]]] = [] for entry in context_management: if not isinstance(entry, dict): continue @@ -2771,7 +2771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: try: - args: object = json.loads(tc.get("function", {}).get("arguments", "{}")) + args: object = _DECODED_JSON.validate_python(json.loads(tc.get("function", {}).get("arguments", "{}"))) if not isinstance(args, Mapping): continue call_id = tc.get("id") @@ -3096,7 +3096,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if split_answer is not None: return litellm.Message(content=json.dumps(split_answer)) try: - payload: Final[object] = json.loads(json_mode_content_str) + payload: Final = _DECODED_JSON.validate_python(json.loads(json_mode_content_str)) except (json.JSONDecodeError, RecursionError): # json decode error does occur, return the original tool response str return litellm.Message(content=json_mode_content_str) @@ -3128,6 +3128,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) +_DECODED_JSON: Final = TypeAdapter(object) _REQUEST_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 4c5ebd60617..bd91ba86faa 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -111,6 +111,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_DECODED_JSON: Final = TypeAdapter(object) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -415,7 +416,7 @@ def _sanitized_anthropic_error(response: httpx.Response, detail: str | None = No if detail is not None: return f"HTTP {response.status_code}: {detail}" try: - body: Final = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) except ValueError: return f"HTTP {response.status_code}" error: Final = body.get("error") if isinstance(body, dict) else None @@ -1700,7 +1701,7 @@ def strip_encrypted_reasoning_blocks_from_anthropic_messages( def strip_thinking_blocks_from_anthropic_messages_request_dict( - data: dict[str, Any], + data: dict[str, object], ) -> None: """ Mutate an Anthropic Messages-style request dict: strip thinking blocks from diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index defb1fa8f38..98cd16c6403 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -1352,12 +1352,12 @@ class LiteLLMAnthropicMessagesAdapter: choices: list[Choices], tool_name_mapping: dict[str, str] | None = None, ) -> list[dict[str, object]]: - new_content: Final[list[dict[str, Any]]] = [] + new_content: Final[list[dict[str, object]]] = [] for choice, compaction_blocks in ( (choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields"))) for choice in choices ): - new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks) + new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) # Handle thinking blocks first if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 9c186670b01..00bf1774dd0 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -9,7 +9,7 @@ import asyncio import contextvars from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from functools import partial -from typing import Any, Final, cast +from typing import Final, cast from pydantic import TypeAdapter @@ -312,7 +312,7 @@ async def anthropic_messages( client: AsyncHTTPHandler | None = None, custom_llm_provider: str | None = None, **kwargs, -) -> AnthropicMessagesResponse | Iterator[bytes] | AsyncIterator[Any]: +) -> AnthropicMessagesResponse | Iterator[bytes] | AsyncIterator[object]: """ Async: Make llm api request in Anthropic /messages API spec. @@ -489,8 +489,8 @@ def anthropic_messages_handler( ) -> ( AnthropicMessagesResponse | Iterator[bytes] - | AsyncIterator[Any] - | Coroutine[Any, Any, AnthropicMessagesResponse | AsyncIterator[Any] | Iterator[bytes]] + | AsyncIterator[object] + | Coroutine[object, None, AnthropicMessagesResponse | AsyncIterator[object] | Iterator[bytes]] ): """ Makes Anthropic `/v1/messages` API calls In the Anthropic API Spec diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py index ae4e18466cf..5fb2546a5ae 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py @@ -91,11 +91,14 @@ def is_anthropic_content_delta_chunk(chunk: object) -> bool: return False +_DECODED_SSE_DATA: Final = TypeAdapter(object) + + def _decoded_sse_data_line(line: bytes) -> object | None: if not line.startswith(b"data:"): return None try: - return json.loads(line[len(b"data:") :].strip()) + return _DECODED_SSE_DATA.validate_python(json.loads(line[len(b"data:") :].strip())) except (ValueError, TypeError): return None diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 4c667574d40..cb96708f200 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -1,5 +1,5 @@ from collections.abc import AsyncIterator, Mapping, Sequence -from typing import Any, ClassVar, Final +from typing import ClassVar, Final import httpx @@ -265,7 +265,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, @@ -293,7 +293,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self, headers: dict, # mutable-ok: mirrors the sync validate_anthropic_messages_environment contract model: str, - messages: list[Any], # mutable-ok: mirrors the sync validate_anthropic_messages_environment contract + messages: list[object], # mutable-ok: mirrors the sync validate_anthropic_messages_environment contract optional_params: dict, # mutable-ok: mirrors the sync validate_anthropic_messages_environment contract litellm_params: dict, # mutable-ok: mirrors the sync validate_anthropic_messages_environment contract api_key: str | None = None, diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/pass_through/messages/utils.py index 36bd10ab8e6..ae5ab68242b 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/pass_through/messages/utils.py @@ -1,10 +1,11 @@ from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache -from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints +from typing import TYPE_CHECKING, Final, cast, get_type_hints from pydantic import JsonValue from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.common_utils import ( flatten_unencrypted_web_search_results_in_anthropic_messages, sanitize_tool_use_ids_in_anthropic_messages, @@ -30,7 +31,7 @@ INCOMPLETE_STREAM_ERROR_MESSAGE: Final = ( ) -def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | None: +def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, object] | None: """ Return the ``stop_details`` of an Anthropic Messages response refused by a safeguard (``stop_reason: "refusal"`` carrying ``stop_details``: @@ -175,7 +176,7 @@ def anthropic_messages_optional_param_keys() -> frozenset[str]: class AnthropicMessagesRequestUtils: @staticmethod def get_requested_anthropic_messages_optional_param( - params: dict[str, Any], + params: Mapping[str, object], *, model: str | None = None, drop_params: bool = False, @@ -222,10 +223,15 @@ class AnthropicMessagesRequestUtils: ) if is_reasoning_auto_summary_enabled(): thinking_param: Final = filtered_params.get("thinking") - if isinstance(thinking_param, dict) and thinking_param.get("type") != "disabled": + thinking_mapping: Final = as_str_mapping(thinking_param) + if ( + isinstance(thinking_param, dict) + and thinking_mapping is not None + and thinking_mapping.get("type") != "disabled" + ): return cast( AnthropicMessagesRequestOptionalParams, - {**filtered_params, "thinking": {**thinking_param, "display": "summarized"}}, + {**filtered_params, "thinking": {**thinking_mapping, "display": "summarized"}}, ) return cast(AnthropicMessagesRequestOptionalParams, filtered_params) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 79dd7f098c8..75f96bfdf63 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -6,7 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Final, Protocol, cast +from typing import Final, Protocol, cast from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES @@ -28,7 +28,15 @@ from ..azure import AzureChatCompletion # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01" -async def forward_messages(client_ws: Any, backend_ws: Any): +class _RealtimeTextSource(Protocol): + async def recv(self) -> str: ... + + +class _RealtimeTextSink(Protocol): + async def send_text(self, data: str) -> None: ... + + +async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTextSource): import websockets try: diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 75f47b9229c..98b4241d7ab 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal import httpx from openai.types.responses import ResponseReasoningItem +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -21,6 +22,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): # Parameters not supported by Azure Responses API @@ -345,7 +348,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): Transform the cancel response API response into a ResponsesAPIResponse """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 6803bf7e551..91c75661688 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -2,6 +2,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication """ +from collections.abc import Sequence from typing import Any, Final from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -29,7 +30,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 3a02a1d9dc2..fee12e34fa3 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -14,6 +14,7 @@ from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountResponse @@ -24,8 +25,8 @@ class BaseTokenCounter(ABC): self, model_to_use: str, messages: list[dict[str, Any]] | None, - contents: list[dict[str, Any]] | None, - deployment: dict[str, Any] | None = None, + contents: list[dict[str, object]] | None, + deployment: dict[str, object] | None = None, request_model: str = "", tools: list[dict[str, Any]] | None = None, system: Any | None = None, @@ -141,8 +142,9 @@ def convert_tool_response_to_message( json_mode_content_str: Final[str | None] = tool_calls[0]["function"].get("arguments") try: if json_mode_content_str is not None: - args: Final = json.loads(json_mode_content_str) - if isinstance(args, dict) and (values := args.get("values")) is not None: + args: Final = _DECODED_JSON.validate_python(json.loads(json_mode_content_str)) + args_object: Final = as_str_mapping(args) + if args_object is not None and (values := args_object.get("values")) is not None: _message = Message(content=json.dumps(values)) return _message else: @@ -170,6 +172,7 @@ class ResponseFormatSchema: return self.allows_undeclared_keys and not self.required_names +_DECODED_JSON: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, JsonValue]) _NAMES: Final = TypeAdapter(frozenset[str]) @@ -224,7 +227,7 @@ def _json_object_or_encoded_object(value: JsonValue) -> Mapping[str, JsonValue] if not isinstance(value, str): return as_json_object(value) try: - return as_json_object(json.loads(value)) + return as_json_object(_DECODED_JSON.validate_python(json.loads(value))) except (json.JSONDecodeError, RecursionError): return None diff --git a/litellm/llms/base_llm/batches/transformation.py b/litellm/llms/base_llm/batches/transformation.py index 34c622d4cf6..e7f3f1d277a 100644 --- a/litellm/llms/base_llm/batches/transformation.py +++ b/litellm/llms/base_llm/batches/transformation.py @@ -1,6 +1,7 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx from httpx import Headers @@ -42,9 +43,10 @@ class BaseBatchesConfig(ABC): @classmethod def get_config(cls): """Get configuration dictionary for this class.""" + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/containers/transformation.py b/litellm/llms/base_llm/containers/transformation.py index 04cbb8bc6d8..49487cddb6f 100644 --- a/litellm/llms/base_llm/containers/transformation.py +++ b/litellm/llms/base_llm/containers/transformation.py @@ -2,6 +2,7 @@ from __future__ import annotations import types from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -45,9 +46,10 @@ class BaseContainerConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( @@ -130,7 +132,7 @@ class BaseContainerConfig(ABC): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: Mapping[str, object] | None = None, ) -> tuple[str, dict]: """Transform the container list request into a URL and params. @@ -206,7 +208,7 @@ class BaseContainerConfig(ABC): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: Mapping[str, object] | None = None, ) -> tuple[str, dict]: """Transform the container file list request into a URL and params. diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 083e396c0bd..4346e214dc8 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -24,10 +24,11 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent + from litellm.types.proxy.auth.user_api_key_auth import Span as _Span from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth as _UserAPIKeyAuth LiteLLMLoggingObj = _LiteLLMLoggingObj - Span = Any + Span = _Span Router = _Router UserAPIKeyAuth = _UserAPIKeyAuth else: diff --git a/litellm/llms/base_llm/google_genai/transformation.py b/litellm/llms/base_llm/google_genai/transformation.py index bd0d29d5ea3..6b1713482b0 100644 --- a/litellm/llms/base_llm/google_genai/transformation.py +++ b/litellm/llms/base_llm/google_genai/transformation.py @@ -1,6 +1,7 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx @@ -30,9 +31,10 @@ class BaseGoogleGenAIGenerateContentConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( @@ -78,7 +80,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC): self, generate_content_config_dict: GenerateContentConfigDict, model: str, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Map Google GenAI parameters to provider-specific format. @@ -161,7 +163,7 @@ class BaseGoogleGenAIGenerateContentConfig(ABC): contents: GenerateContentContentListUnionDict, tools: ToolConfigDict | None, generate_content_config_dict: dict, - system_instruction: Any | None = None, + system_instruction: object | None = None, ) -> dict: """ Transform the request parameters for the generate content API. diff --git a/litellm/llms/base_llm/managed_resources/utils.py b/litellm/llms/base_llm/managed_resources/utils.py index b6919fa561b..56a62da39a9 100644 --- a/litellm/llms/base_llm/managed_resources/utils.py +++ b/litellm/llms/base_llm/managed_resources/utils.py @@ -7,13 +7,13 @@ different managed resource types (files, vector stores, etc.). import base64 import re -from typing import Any, Final, Literal +from typing import Final, Literal PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS: Final = ("azure", "azure_ai") def resolve_passthrough_managed_id_provider( - custom_llm_provider: Any, + custom_llm_provider: object, ) -> str | None: """Map a pass-through ``custom_llm_provider`` to the provider scope that namespaces passthrough managed object IDs, or ``None`` when the route is not @@ -29,7 +29,8 @@ def resolve_passthrough_managed_id_provider( Splitting them would make a managed ID minted on ``azure`` fail to resolve when replayed on ``azure_ai`` and vice versa. """ - provider: Final = str(getattr(custom_llm_provider, "value", custom_llm_provider) or "").lower() + provider_value: Final[object] = getattr(custom_llm_provider, "value", custom_llm_provider) + provider: Final = str(provider_value or "").lower() if not provider: return None if provider in PASSTHROUGH_MANAGED_ID_AZURE_PROVIDERS or provider.endswith((".azure", ".azure_ai")): diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index a03cd0df913..ed8bfc8cb96 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -38,9 +38,10 @@ class BaseResponsesAPIConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/vector_store_files/transformation.py b/litellm/llms/base_llm/vector_store_files/transformation.py index 9fb4d3e9dac..79943c857d9 100644 --- a/litellm/llms/base_llm/vector_store_files/transformation.py +++ b/litellm/llms/base_llm/vector_store_files/transformation.py @@ -53,7 +53,7 @@ class BaseVectorStoreFilesConfig(ABC): return optional_params @abstractmethod - def get_auth_credentials(self, litellm_params: dict[str, Any]) -> VectorStoreFileAuthCredentials: ... + def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> VectorStoreFileAuthCredentials: ... @abstractmethod def get_vector_store_file_endpoints_by_type( @@ -75,7 +75,7 @@ class BaseVectorStoreFilesConfig(ABC): *, api_base: str | None, vector_store_id: str, - litellm_params: dict[str, Any], + litellm_params: Mapping[str, object], ) -> str: if api_base is None: raise ValueError("api_base is required") @@ -183,7 +183,7 @@ class BaseVectorStoreFilesConfig(ABC): *, error_message: str, status_code: int, - headers: dict[str, Any] | httpx.Headers, + headers: dict[str, str] | httpx.Headers, ) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index b1b5f50fbe6..d7b643eb7f4 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import FileContent, RequestFiles @@ -35,9 +35,10 @@ class BaseVideoConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 91d18c651da..848120150cd 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -613,7 +613,10 @@ class BaseAWSLLM(SignsRequestsWithAWS): if provider is not None: return provider - for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL): + invoke_providers: Final[tuple[BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]] = get_args( + BEDROCK_INVOKE_PROVIDERS_LITERAL + ) + for provider in invoke_providers: if provider in model: return provider return None @@ -726,9 +729,12 @@ class BaseAWSLLM(SignsRequestsWithAWS): return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider) # Fallback: check if any provider name appears in the model string - for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): + embedding_providers: Final[tuple[BEDROCK_EMBEDDING_PROVIDERS_LITERAL, ...]] = get_args( + BEDROCK_EMBEDDING_PROVIDERS_LITERAL + ) + for provider in embedding_providers: if provider in model: - return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider) + return provider return None diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 021327eaa1f..35d08d4b2c2 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -1,5 +1,6 @@ import json -from typing import Any, Final +from collections.abc import Iterator +from typing import Final import httpx @@ -35,7 +36,7 @@ def make_sync_call( fake_stream: bool = False, stream_chunk_size: int | None = None, timeout: float | httpx.Timeout | None = None, -) -> tuple[Any, httpx.Headers]: +) -> tuple[MockResponseIterator | Iterator[object], httpx.Headers]: if client is None: client = get_httpx_client() # Create a new client if none provided @@ -69,7 +70,9 @@ def make_sync_call( messages=messages, encoding=litellm.encoding, ) - completion_stream: Any = MockResponseIterator(model_response=model_response, json_mode=json_mode) + completion_stream: MockResponseIterator | Iterator[object] = MockResponseIterator( + model_response=model_response, json_mode=json_mode + ) else: decoder: Final = AWSEventStreamDecoder(model=model, json_mode=json_mode) completion_stream = decoder.iter_bytes( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cfbb9956988..b6264f664d3 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -145,6 +145,8 @@ UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS: Final = [ _TOOLS_AS_SENT: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_JSON_VALUE: Final = TypeAdapter(object) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) _TOOLS_AS_RECEIVED: Final = TypeAdapter(list[object]) CLAUDE_ON_BEDROCK_INVOKE_REMEDY: Final = " This model runs tool search on Bedrock through `bedrock/invoke/`." _DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -327,9 +329,10 @@ class AmazonConverseConfig(BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("_") and not isinstance( v, @@ -2586,12 +2589,13 @@ class AmazonConverseConfig(BaseConfig): Unwrapped JSON string or original if unwrapping not needed """ try: - response_data = json.loads(json_str) - if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1: - response_data = response_data["properties"] - return json.dumps(response_data) + response_data: Final = _JSON_VALUE.validate_python(json.loads(json_str)) except json.JSONDecodeError: - pass + return json_str + if isinstance(response_data, dict): + json_object: Final = _JSON_OBJECT.validate_python(response_data) + if "properties" in json_object and len(json_object) == 1: + return json.dumps(json_object["properties"]) return json_str @staticmethod diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 431b09c0edd..a207bceb418 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -116,16 +116,17 @@ class AmazonCohereChatConfig: stop_sequences: str | None = None, raw_prompting: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, @@ -830,7 +831,7 @@ class AWSEventStreamDecoder: def iter_bytes( self, iterator: Iterator[bytes], *, response_headers: Mapping[str, str] | None = None - ) -> Iterator[GChunk | ModelResponseStream | dict]: + ) -> Iterator[GChunk | ModelResponseStream | dict[str, object]]: """Given an iterator that yields lines, iterate over it & yield every event encountered""" from botocore.eventstream import EventStreamBuffer diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 41e539e52fd..16dde81ca47 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Final, cast from httpx import Response @@ -80,7 +80,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig): class AmazonDeepseekR1ResponseIterator(BaseModelResponseIterator): - def __init__(self, streaming_response: Any, sync_stream: bool) -> None: + def __init__(self, streaming_response: object, sync_stream: bool) -> None: super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) self.has_finished_thinking = False diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index f4867f8dfc0..f7566ac4923 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -20,6 +20,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation AmazonInvokeConfig, ) from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.custom_httpx.http_handler import header_value from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_base64_str @@ -245,8 +246,8 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): ) # Calculate usage from headers - bedrock_input_tokens: Final = raw_response.headers.get("x-amzn-bedrock-input-token-count", None) - bedrock_output_tokens: Final = raw_response.headers.get("x-amzn-bedrock-output-token-count", None) + bedrock_input_tokens: Final = header_value(raw_response.headers, "x-amzn-bedrock-input-token-count") + bedrock_output_tokens: Final = header_value(raw_response.headers, "x-amzn-bedrock-output-token-count") prompt_tokens: Final = int(bedrock_input_tokens or litellm.token_counter(messages=messages)) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 2356efe0f48..1b3b0c344df 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -56,6 +56,10 @@ _INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LI ) +def _response_header(headers: Mapping[str, str], name: str) -> str | None: + return headers.get(name, None) + + def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]": try: guardrail_config: Final = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config) @@ -398,8 +402,8 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): ) ## CALCULATING USAGE - bedrock returns usage in the headers - bedrock_input_tokens: Final = raw_response.headers.get("x-amzn-bedrock-input-token-count", None) - bedrock_output_tokens: Final = raw_response.headers.get("x-amzn-bedrock-output-token-count", None) + bedrock_input_tokens: Final = _response_header(raw_response.headers, "x-amzn-bedrock-input-token-count") + bedrock_output_tokens: Final = _response_header(raw_response.headers, "x-amzn-bedrock-output-token-count") prompt_tokens: Final = int(bedrock_input_tokens or litellm.token_counter(messages=messages)) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 6f8ec6dbd4a..cea1140bf65 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -1,5 +1,5 @@ from collections.abc import AsyncIterator, Iterator, Mapping -from typing import Any, Final +from typing import TYPE_CHECKING, Final import litellm from litellm.llms.anthropic.chat.transformation import AnthropicConfig, JsonModeRequest @@ -13,6 +13,9 @@ from .common_utils import ( resolve_unsupported_override, ) +if TYPE_CHECKING: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): """ @@ -96,7 +99,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "ModelResponseIterator": return self.get_model_response_iterator_for_request( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, optional_params={} ) @@ -107,7 +110,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): sync_stream: bool, json_mode: bool | None, optional_params: Mapping[str, object], - ) -> object: + ) -> "ModelResponseIterator": from litellm.llms.anthropic.chat.handler import ModelResponseIterator return ModelResponseIterator( diff --git a/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py index db0dc7e65c2..1cece8743de 100644 --- a/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py +++ b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py @@ -17,6 +17,7 @@ import os from typing import TYPE_CHECKING, Any, Final, cast import httpx +from httpx._types import RequestFiles from litellm._logging import verbose_logger from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig @@ -319,7 +320,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig): image_edit_optional_request_params: dict, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[dict, Any]: + ) -> tuple[dict, RequestFiles]: op: Final = dict(image_edit_optional_request_params) image_b64: Final = _file_types_to_b64(image) diff --git a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py index 63725473923..6f972986e56 100644 --- a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Any, Final from openai.types.image import Image @@ -26,9 +27,10 @@ class AmazonNovaCanvasConfig: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py index df91e736239..3ebb128e89c 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py @@ -1,6 +1,7 @@ import copy import os import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -52,16 +53,17 @@ class AmazonStabilityConfig: width: int | None = None, height: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index a1a984f35a1..8c5ef7138ad 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -103,7 +103,7 @@ class AmazonAnthropicClaudeMessagesConfig( self, headers: dict, model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, @@ -878,7 +878,7 @@ class AmazonAnthropicClaudeMessagesConfig( chunk_type = chunk.get("type") if chunk_type == "message_start": - msg: dict[str, object] = cast(dict[str, Any], chunk.get("message") or {}) + msg: dict[str, object] = cast(dict[str, object], chunk.get("message") or {}) u = msg.get("usage") if isinstance(u, dict): start_usage_snapshot = dict(u) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 0b62d733bb6..03e2ee6745c 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -1,5 +1,6 @@ import json from collections.abc import Callable, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional, cast import httpx @@ -39,7 +40,8 @@ def _plain_text_delta(chunk: ModelResponseStream) -> str | None: choice: Final = chunk.choices[0] if choice.finish_reason or choice.logprobs is not None: return None - populated: Final = frozenset(key for key, value in choice.delta.model_dump().items() if value is not None) + delta_fields: Final[Mapping[str, object]] = MappingProxyType(choice.delta.model_dump()) + populated: Final = frozenset(key for key, value in delta_fields.items() if value is not None) if not populated <= _TEXT_ONLY_DELTA_FIELDS: return None content: Final = choice.delta.get("content") diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 8fef84bae29..3038b0831f4 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -7,9 +7,11 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format. import base64 import json import uuid as uuid_lib +from collections.abc import Mapping from typing import Final, cast import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -44,6 +46,9 @@ from litellm.types.realtime import ( RealtimeResponseTypedDict, ) +_JSON_VALUE: Final = TypeAdapter(object) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) + class BedrockContentEnd(LiteLLMBaseModel): stopReason: str | None = None @@ -839,10 +844,10 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not isinstance(additional_model_fields, str): return None try: - parsed: Final = json.loads(additional_model_fields) + parsed: Final = _JSON_VALUE.validate_python(json.loads(additional_model_fields)) except json.JSONDecodeError: return None - stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None + stage: Final = _JSON_OBJECT.validate_python(parsed).get("generationStage") if isinstance(parsed, dict) else None return stage if isinstance(stage, str) else None def _current_user_item_id(self, new_utterance: bool = False) -> str: diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index 524926251d9..1c791e994f2 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -11,7 +11,7 @@ Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the """ from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -26,12 +26,18 @@ from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_serie from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ModelResponse from ...base_llm.chat.transformation import BaseLLMException from ...bedrock.common_utils import BedrockError from ...openai_like.chat.transformation import OpenAILikeChatConfig from ..common_utils import mantle_base_segment, split_mantle_region_prefix +if TYPE_CHECKING: + from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIChatCompletionStreamingHandler, + ) + class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): """ @@ -161,10 +167,10 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): def get_model_response_iterator( self, - streaming_response: Iterator[str] | AsyncIterator[str] | Any, + streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "OpenAIChatCompletionStreamingHandler": from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, ) diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index e107674b462..c297eb13db1 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -2,6 +2,7 @@ import json import time import traceback from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -44,7 +45,7 @@ class BytezChatConfig(BaseConfig): def __init__( self, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 37c53640d18..41b1569bc4a 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -1,5 +1,6 @@ import time -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -93,7 +94,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): tool_results: list | None = None, seed: int | None = None, ) -> None: - locals_: Final = locals() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/cohere/embed/transformation.py b/litellm/llms/cohere/embed/transformation.py index 73b58254a3d..590164ffa0f 100644 --- a/litellm/llms/cohere/embed/transformation.py +++ b/litellm/llms/cohere/embed/transformation.py @@ -33,6 +33,7 @@ from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class _SupportsEncode(Protocol): @@ -183,7 +184,7 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig): encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: - response_json: Final = response.json() + response_json: Final = _DECODED_JSON.validate_python(response.json()) ## LOGGING logging_obj.post_call( input=input, diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index d690fd9ddde..447d7b685f7 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -440,7 +440,7 @@ class BaseLLMAIOHTTPHandler: timeout: float | httpx.Timeout, fake_stream: bool = False, client: HTTPHandler | None = None, - ) -> tuple[Any, dict]: + ) -> tuple[object, dict]: if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = get_httpx_client() else: diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index 5f2727e3e5a..bfa163c7546 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -55,6 +55,7 @@ class EndpointsConfig(TypedDict): _ENDPOINTS_CONFIG: Final = TypeAdapter(EndpointsConfig) +_DECODED_JSON: Final = TypeAdapter(object) class ContainerErrorDetail(TypedDict, total=False): @@ -145,7 +146,7 @@ def _build_query_params( def error_message_from_response(response: httpx.Response) -> str: try: - body: Final[object] = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) except ValueError: return response.text diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 629fd4e2b3a..cffeb44f12c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -797,9 +797,11 @@ class AsyncHTTPHandler: ) if response.is_redirect or response.is_error: return httpx.Response(response.status_code, headers=response.headers, request=response.request) - if response.headers.get("content-encoding", "identity").lower() != "identity": + content_encoding: Final = header_value(response.headers, "content-encoding") + if ("identity" if content_encoding is None else content_encoding).lower() != "identity": raise HTTPResponseLimitError("Response size limits require an uncompressed response") - if int(response.headers.get("content-length", "0")) > max_bytes: + content_length: Final = header_value(response.headers, "content-length") + if int("0" if content_length is None else content_length) > max_bytes: raise HTTPResponseLimitError("Response exceeds the configured size limit") with BytesIO() as body: async for chunk in response.aiter_bytes(chunk_size=65536): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 28396f070d1..610321fe1f6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -109,6 +109,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, get_httpx_client, + header_value, ) from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, @@ -473,8 +474,8 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers: `aiter_bytes` yields the decoded body, so the upstream transfer headers only describe the bytes on the wire when no content-encoding was applied. """ - headers: Final[Mapping[str, str]] = response.headers - if headers.get("content-encoding", "identity").lower() == "identity": + content_encoding: Final = header_value(response.headers, "content-encoding") + if content_encoding is None or content_encoding.lower() == "identity": return response.headers return httpx.Headers( [ @@ -2556,9 +2557,10 @@ class BaseLLMHTTPHandler: **{key: value for key, value in response_api_optional_request_params.items() if key in modified_kwargs}, **{key: value for key, value in modified_kwargs.items() if key in optional_param_names}, } + current_litellm_params: Final[Mapping[str, object]] = dict(litellm_params) updated_litellm_params: Final = GenericLiteLLMParams( **{ - **{key: value for key, value in dict(litellm_params).items() if key in modified_kwargs}, + **{key: value for key, value in current_litellm_params.items() if key in modified_kwargs}, **{ key: value for key, value in modified_kwargs.items() @@ -3522,9 +3524,7 @@ class BaseLLMHTTPHandler: """ if upload_url_location == "headers": # Google Cloud Storage style - URL in X-Goog-Upload-URL header - upload_headers: Final[Mapping[str, str]] = response.headers - upload_url = upload_headers.get("X-Goog-Upload-URL") - return upload_url, None + return header_value(response.headers, "X-Goog-Upload-URL"), None else: # Response body style (e.g., Manus, S3 presigned URLs) try: @@ -6344,7 +6344,7 @@ class BaseLLMHTTPHandler: error_text = error.response.text else: error_text = getattr(error, "text", str(error)) - error_response: Final = getattr(error, "response", None) + error_response: Final[object] = getattr(error, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) if error_response and hasattr(error_response, "text"): diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index e32e28fb80e..2c04b32b1da 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -4,6 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` to Databricks' `/chat/completion import os from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx @@ -196,7 +197,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): stop: list[str] | str | None = None, n: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/deepagents/harness/transformation.py b/litellm/llms/deepagents/harness/transformation.py index cdac6944019..cb80e3e12f7 100644 --- a/litellm/llms/deepagents/harness/transformation.py +++ b/litellm/llms/deepagents/harness/transformation.py @@ -211,11 +211,12 @@ def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event, calls = getattr(message, "tool_calls", None) or () return tuple(tool_call_event(call) for call in calls if call.get("name") not in skip_tools) if kind == "tool" and getattr(message, "name", None) not in skip_tools: + status: Final[object] = getattr(message, "status", None) return ( ToolResult( id=str(getattr(message, "tool_call_id", "") or ""), output=content_text(getattr(message, "content", "")), - is_error=getattr(message, "status", None) == "error", + is_error=status == "error", ), ) return () diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index ad58070df4a..6a6b57131ad 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -2,6 +2,7 @@ DeepSeek Anthropic-compatible messages transformation config. """ +from collections.abc import Sequence from typing import Any, Final import litellm @@ -46,7 +47,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index 8f5c35f32f2..cb56717993e 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -1,7 +1,7 @@ import json import time import types -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Final import httpx @@ -140,16 +140,17 @@ class AlephAlphaConfig: contextual_control_threshold: int | None = None, control_log_additive: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls) -> dict[str, object]: + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/fal_ai/image_generation/bytedance_transformation.py b/litellm/llms/fal_ai/image_generation/bytedance_transformation.py index 44cf816c1aa..de7b3c9b958 100644 --- a/litellm/llms/fal_ai/image_generation/bytedance_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bytedance_transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig @@ -23,7 +24,7 @@ class FalAIBytedanceBaseConfig(FalAIFluxProV11UltraConfig): def map_openai_params( self, - non_default_params: dict, + non_default_params: Mapping[str, object], optional_params: dict, model: str, drop_params: bool, @@ -60,7 +61,7 @@ class FalAIBytedanceBaseConfig(FalAIFluxProV11UltraConfig): return optional_params - def _map_image_size(self, size: Any) -> object: + def _map_image_size(self, size: object) -> object: if isinstance(size, dict): return size diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_transformation.py index 48b5d392747..f9b7c9440ee 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig @@ -28,7 +29,7 @@ class FalAIFluxProV11Config(FalAIFluxProV11UltraConfig): def map_openai_params( self, - non_default_params: dict, + non_default_params: Mapping[str, object], optional_params: dict, model: str, drop_params: bool, @@ -68,7 +69,7 @@ class FalAIFluxProV11Config(FalAIFluxProV11UltraConfig): return optional_params - def _map_image_size(self, size: Any) -> object: + def _map_image_size(self, size: object) -> object: if isinstance(size, dict): return size if not isinstance(size, str): diff --git a/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py b/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py index ed26cc4bc12..1e87714c5cb 100644 --- a/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig @@ -28,7 +29,7 @@ class FalAIFluxSchnellConfig(FalAIFluxProV11UltraConfig): def map_openai_params( self, - non_default_params: dict, + non_default_params: Mapping[str, object], optional_params: dict, model: str, drop_params: bool, @@ -65,7 +66,7 @@ class FalAIFluxSchnellConfig(FalAIFluxProV11UltraConfig): return optional_params - def _map_image_size(self, size: Any) -> object: + def _map_image_size(self, size: object) -> object: if isinstance(size, dict): return size diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index ff56f3473d5..1e7b0d126ab 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -19,6 +19,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIIdeogramV3Config(FalAIBaseConfig): @@ -57,7 +58,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): def map_openai_params( self, - non_default_params: dict, + non_default_params: Mapping[str, object], optional_params: dict, model: str, drop_params: bool, @@ -93,7 +94,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): return optional_params - def _map_image_size(self, size: Any) -> object: + def _map_image_size(self, size: object) -> object: if isinstance(size, dict): width = size.get("width") height = size.get("height") @@ -163,7 +164,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index d0551c22441..8b6946ee562 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -58,6 +58,7 @@ if TYPE_CHECKING: _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) def _map_reasoning_effort(value: object) -> object: @@ -745,7 +746,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): ## RESPONSE OBJECT try: - completion_response: Final = raw_response.json() + completion_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: response_headers: Final = getattr(raw_response, "headers", None) raise FireworksAIException( diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index 07e05b2b295..1c9d09330a8 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -96,7 +96,7 @@ class GoogleAIStudioTokenCounter: headers: dict[str, object] | None = None, model: str = "", litellm_params: dict[str, object] | None = None, - ) -> tuple[dict[str, Any], str]: + ) -> tuple[dict[str, object], str]: """ Returns a Tuple of headers and url for the Google Gen AI Studio countTokens endpoint. """ diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index 2eb84423633..196a19a141f 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -101,7 +101,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): self, generate_content_config_dict: GenerateContentConfigDict, model: str, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Map Google GenAI parameters to provider-specific format. @@ -337,7 +337,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): contents: GenerateContentContentListUnionDict, tools: ToolConfigDict | None, generate_content_config_dict: dict, - system_instruction: Any | None = None, + system_instruction: object | None = None, ) -> dict: from litellm.types.google_genai.main import ( GenerateContentConfigDict, diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index f6525a449b6..34bd9660c43 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -208,18 +208,20 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): else: file_search_config["metadata_filter"] = metadata_filter - # Build request body - request_body: Final[dict[str, Any]] = { - "contents": [{"parts": [{"text": query}]}], - "tools": [{"file_search": file_search_config}], - } - # Add max_num_results if specified max_results: Final = vector_store_search_optional_params.get("max_num_results") - if max_results: - # This might need to be added to generationConfig or tool config - # depending on Gemini's API requirements - request_body.setdefault("generationConfig", {})["candidateCount"] = 1 + # This might need to be added to generationConfig or tool config + # depending on Gemini's API requirements + generation_config: Final[Mapping[str, object]] = ( + {"generationConfig": {"candidateCount": 1}} if max_results else {} + ) + + # Build request body + request_body: Final[dict[str, object]] = { + "contents": [{"parts": [{"text": query}]}], + "tools": [{"file_search": file_search_config}], + **generation_config, + } litellm_logging_obj.model_call_details["query"] = query litellm_logging_obj.model_call_details["vector_store_id"] = vector_store_id diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 15a1b463c3f..1fe7de0ba85 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -14,6 +14,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -35,6 +36,7 @@ else: _EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({}) +_JSON_VALUE: Final = TypeAdapter(object) def is_valid_json(value: str) -> bool: @@ -392,7 +394,7 @@ class GigaChatConfig(BaseConfig): args = func.get("arguments", "{}") if isinstance(args, str): try: - args = json.loads(args) + args = _JSON_VALUE.validate_python(json.loads(args)) except json.JSONDecodeError: args = {} message["function_call"] = { diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index 321f5525433..e0e746427ad 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -43,9 +43,10 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index a6250afdd03..7f9209afd65 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -20,6 +20,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.types.llms.openai import AllEmbeddingInputValues from litellm.types.utils import EmbeddingResponse @@ -204,7 +205,7 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): optional_params[param] = value return optional_params - def get_error_class(self, error_message: str, status_code: int, headers: Any) -> Any: + def get_error_class(self, error_message: str, status_code: int, headers: Any) -> BaseLLMException: from litellm.llms.openai.openai import OpenAIConfig return OpenAIConfig().get_error_class(error_message=error_message, status_code=status_code, headers=headers) diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 6980a25e2eb..8c269cb5dfd 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Groq's `/v1/chat/completions` """ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, overload import httpx from pydantic import BaseModel, TypeAdapter, ValidationError @@ -91,7 +91,7 @@ class GroqChatConfig(OpenAILikeChatConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "GroqChatCompletionStreamingHandler": return GroqChatCompletionStreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, @@ -324,8 +324,9 @@ class GroqChatConfig(OpenAILikeChatConfig): json_mode=json_mode, ) + original_service_tier: Final[object] = getattr(model_response, "service_tier") mapped_service_tier: Final[Literal["auto", "default", "flex"]] = self._map_groq_service_tier( - original_service_tier=getattr(model_response, "service_tier") + original_service_tier=original_service_tier ) setattr(model_response, "service_tier", mapped_service_tier) self._add_web_search_usage(model_response=model_response) @@ -356,7 +357,7 @@ class GroqChatConfig(OpenAILikeChatConfig): verbose_logger.info("Groq executed_tools entries did not match the expected shape; not billed: %s", e) return () - def _map_groq_service_tier(self, original_service_tier: str | None) -> Literal["auto", "default", "flex"]: + def _map_groq_service_tier(self, original_service_tier: object) -> Literal["auto", "default", "flex"]: """ Ensure groq service tier is OpenAI compatible. """ diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 764d80c6f82..a463fc1d3c9 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -4,7 +4,7 @@ Transformation logic for Hosted VLLM rerank from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Final +from typing import Final import httpx from pydantic import ValidationError @@ -86,7 +86,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 334fc9a5d9a..0883a9a8077 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -1,7 +1,7 @@ import json import os import time -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from copy import deepcopy from typing import TYPE_CHECKING, Any, Final, Protocol @@ -39,6 +39,10 @@ class _TokenEncoding(Protocol): def encode(self, text: str, /) -> Sequence[object]: ... +def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: + return headers.get(key, default) + + tgi_models_cache = None conv_models_cache = None @@ -485,7 +489,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): ## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten) task: Final = litellm_params.get("task", None) is_streamed = False - if raw_response.__dict__["headers"].get("Content-Type", "") == "text/event-stream": + if _header_value(raw_response.headers, "Content-Type", "") == "text/event-stream": is_streamed = True # iterate over the complete streamed response, and return the final answer diff --git a/litellm/llms/jina_ai/embedding/transformation.py b/litellm/llms/jina_ai/embedding/transformation.py index 9ac579523ea..9b156e5ae21 100644 --- a/litellm/llms/jina_ai/embedding/transformation.py +++ b/litellm/llms/jina_ai/embedding/transformation.py @@ -8,7 +8,7 @@ Docs - https://jina.ai/embeddings/ import types from collections.abc import Mapping -from typing import Final, cast +from typing import Final, Protocol, cast import httpx from pydantic import ConfigDict, TypeAdapter @@ -27,6 +27,14 @@ from ..common_utils import JinaAIError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class JinaAIEmbeddingConfig(BaseEmbeddingConfig): """ Reference: https://jina.ai/embeddings/ @@ -42,9 +50,10 @@ class JinaAIEmbeddingConfig(BaseEmbeddingConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, @@ -147,7 +156,7 @@ class JinaAIEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict, litellm_params: dict, ) -> EmbeddingResponse: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) ## LOGGING logging_obj.post_call( input=input, diff --git a/litellm/llms/langflow/a2a.py b/litellm/llms/langflow/a2a.py index 060dc0a4d05..1d8044e7248 100644 --- a/litellm/llms/langflow/a2a.py +++ b/litellm/llms/langflow/a2a.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from litellm.a2a_protocol.utils import ( get_session_id_from_a2a_params, @@ -7,10 +8,10 @@ from litellm.a2a_protocol.utils import ( def merge_a2a_session_into_litellm_params( - litellm_params: dict[str, Any], - params: dict[str, Any], + litellm_params: Mapping[str, object], + params: Mapping[str, object], principal: str | None = None, -) -> dict[str, Any]: +) -> dict[str, object]: merged: Final = dict(litellm_params) session_id: Final = get_session_id_from_a2a_params(params) if session_id and "session_id" not in merged: diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 82712af9af3..a740050dd94 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio """ from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol from urllib.parse import quote import httpx @@ -28,6 +28,14 @@ _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=Config _STR: Final = TypeAdapter(str) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class LemonadeChatConfig(OpenAILikeChatConfig): _DEFAULT_API_KEY = "lemonade" @@ -189,7 +197,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): headers=self._get_auth_headers(api_key), ) response.raise_for_status() - model_info: Final = response.json() + model_info: Final = _json_body(response) except Exception: verbose_logger.debug("LemonadeError: Could not get model info.") return self._get_default_model_info(model) diff --git a/litellm/llms/litellm_proxy/skills/prompt_injection.py b/litellm/llms/litellm_proxy/skills/prompt_injection.py index 414803b13e7..3377338a03e 100644 --- a/litellm/llms/litellm_proxy/skills/prompt_injection.py +++ b/litellm/llms/litellm_proxy/skills/prompt_injection.py @@ -7,6 +7,7 @@ and injection into the system prompt for non-Anthropic models. import posixpath import zipfile +from collections.abc import Mapping from io import BytesIO from typing import Any, Final @@ -183,7 +184,7 @@ class SkillPromptInjectionHandler: data["messages"] = messages return data - def create_execute_code_tool(self, skill_modules: list[str]) -> dict[str, Any]: + def create_execute_code_tool(self, skill_modules: list[str]) -> Mapping[str, object]: """ Create the execute_code tool definition. @@ -226,7 +227,7 @@ class SkillPromptInjectionHandler: }, } - def convert_skill_to_tool(self, skill: LiteLLM_SkillsTable) -> dict[str, Any]: + def convert_skill_to_tool(self, skill: LiteLLM_SkillsTable) -> Mapping[str, object]: """ Convert a LiteLLM skill to an OpenAI-style tool. @@ -271,7 +272,7 @@ class SkillPromptInjectionHandler: return tool - def convert_skill_to_anthropic_tool(self, skill: LiteLLM_SkillsTable) -> dict[str, Any]: + def convert_skill_to_anthropic_tool(self, skill: LiteLLM_SkillsTable) -> Mapping[str, object]: """ Convert a LiteLLM skill to an Anthropic-style tool (messages API format). diff --git a/litellm/llms/mistral/ocr/guardrail_translation/handler.py b/litellm/llms/mistral/ocr/guardrail_translation/handler.py index 9f4931ee579..e85004dcf2f 100644 --- a/litellm/llms/mistral/ocr/guardrail_translation/handler.py +++ b/litellm/llms/mistral/ocr/guardrail_translation/handler.py @@ -5,7 +5,8 @@ Provides guardrail translation support for the OCR endpoint. Processes the extracted markdown text from OCR pages. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -35,7 +36,7 @@ class OCRHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process OCR input by applying guardrails to the document reference. diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 584cecc0f36..3b70732f3fb 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -75,6 +75,7 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -> str | None: @@ -628,7 +629,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): return None tool_call_names: Final = get_tool_call_names(optional_params.get("tools", [])) try: - json_content: Final[object] = json.loads(content) + json_content: Final[object] = _DECODED_JSON.validate_python(json.loads(content)) if ( isinstance(json_content, dict) and json_content.get("type") == "function" @@ -895,7 +896,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> object: return OpenAIChatCompletionStreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index bf8b42efe4c..c168c3f88fc 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -1012,7 +1012,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): tool_calls_to_check.append(tool_call_dict) tool_call_task_mappings.append((choice_idx, int(tool_call_idx))) - def _convert_tool_call_to_dict(self, tool_call: dict[str, Any] | Any) -> dict[str, Any] | None: + def _convert_tool_call_to_dict(self, tool_call: object) -> dict[str, object] | None: """ Convert a tool call object to dictionary format. @@ -1022,19 +1022,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return tool_call elif hasattr(tool_call, "id") and hasattr(tool_call, "function"): # Convert object to dict - function: Final = tool_call.function - function_dict: Final = {} - if hasattr(function, "name"): - function_dict["name"] = function.name - if hasattr(function, "arguments"): - function_dict["arguments"] = function.arguments + function: Final[object] = getattr(tool_call, "function", None) + function_dict: Final[Mapping[str, object]] = { + key: getattr(function, key, None) for key in ("name", "arguments") if hasattr(function, key) + } - tool_call_dict: Final = { - "id": tool_call.id if hasattr(tool_call, "id") else None, - "type": tool_call.type if hasattr(tool_call, "type") else "function", + return { + "id": getattr(tool_call, "id", None), + "type": getattr(tool_call, "type", "function"), "function": function_dict, } - return tool_call_dict return None async def _apply_guardrail_responses_to_output_texts( diff --git a/litellm/llms/openai/completion/guardrail_translation/handler.py b/litellm/llms/openai/completion/guardrail_translation/handler.py index f3557d4017e..c4bc04d4627 100644 --- a/litellm/llms/openai/completion/guardrail_translation/handler.py +++ b/litellm/llms/openai/completion/guardrail_translation/handler.py @@ -5,7 +5,8 @@ This module provides guardrail translation support for OpenAI's text completion The handler processes the 'prompt' parameter for guardrails. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -14,6 +15,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import TextCompletionResponse @@ -35,7 +37,7 @@ class OpenAITextCompletionHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process input prompt by applying guardrails to text content. @@ -122,9 +124,9 @@ class OpenAITextCompletionHandler(BaseTranslation): response: "TextCompletionResponse", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - user_api_key_dict: Any | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, request_data: dict | None = None, - ) -> Any: + ) -> "TextCompletionResponse": """ Process output response by applying guardrails to completion text. diff --git a/litellm/llms/openai/embeddings/guardrail_translation/handler.py b/litellm/llms/openai/embeddings/guardrail_translation/handler.py index ef464e8a849..c5ed89ecd59 100644 --- a/litellm/llms/openai/embeddings/guardrail_translation/handler.py +++ b/litellm/llms/openai/embeddings/guardrail_translation/handler.py @@ -5,7 +5,8 @@ This module provides guardrail translation support for OpenAI's embeddings endpo The handler processes the 'input' parameter for guardrails. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -14,6 +15,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import EmbeddingResponse @@ -37,7 +39,7 @@ class OpenAIEmbeddingsHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process input text by applying guardrails to text content. @@ -146,9 +148,9 @@ class OpenAIEmbeddingsHandler(BaseTranslation): response: "EmbeddingResponse", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - user_api_key_dict: Any | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, request_data: dict | None = None, - ) -> Any: + ) -> "EmbeddingResponse": """ Process output response - embeddings responses contain vectors, not text. diff --git a/litellm/llms/openai/image_generation/guardrail_translation/handler.py b/litellm/llms/openai/image_generation/guardrail_translation/handler.py index b1d64fb1c09..cce9272017d 100644 --- a/litellm/llms/openai/image_generation/guardrail_translation/handler.py +++ b/litellm/llms/openai/image_generation/guardrail_translation/handler.py @@ -5,7 +5,8 @@ This module provides guardrail translation support for OpenAI's image generation The handler processes the 'prompt' parameter for guardrails. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -14,6 +15,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.utils import ImageResponse @@ -34,7 +36,7 @@ class OpenAIImageGenerationHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process input prompt by applying guardrails to text content. @@ -84,9 +86,9 @@ class OpenAIImageGenerationHandler(BaseTranslation): response: "ImageResponse", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - user_api_key_dict: Any | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, request_data: dict | None = None, - ) -> Any: + ) -> "ImageResponse": """ Process output response - typically not needed for image generation. diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 58eadeadc1e..27346ee936e 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -76,14 +76,14 @@ class MistralEmbeddingConfig: def __init__( self, ) -> None: - locals_: Final[Mapping[str, object]] = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): - config_attrs: Final[Mapping[str, object]] = cls.__dict__ + config_attrs: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v for k, v in config_attrs.items() @@ -169,7 +169,7 @@ class OpenAIConfig(BaseConfig): top_p: int | None = None, response_format: dict | None = None, ) -> None: - locals_: Final[Mapping[str, object]] = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @@ -918,7 +918,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) error_text: Final = getattr(e, "text", str(e)) - error_response: Final = getattr(e, "response", None) + error_response: Final[object] = getattr(e, "response", None) error_body: Final = getattr(e, "body", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) @@ -1041,7 +1041,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise e # e.message except Exception as e: - exception_response = getattr(e, "response", None) + exception_response: object = getattr(e, "response", None) status_code = getattr(e, "status_code", 500) exception_body = getattr(e, "body", None) error_headers = getattr(e, "headers", None) @@ -1201,7 +1201,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): error_headers = getattr(e, "headers", None) status_code = getattr(e, "status_code", 500) - error_response = getattr(e, "response", None) + error_response: object = getattr(e, "response", None) exception_body = getattr(e, "body", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) @@ -1348,7 +1348,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) error_text: Final = getattr(e, "text", str(e)) - error_response: Final = getattr(e, "response", None) + error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) @@ -1438,7 +1438,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) error_text: Final = getattr(e, "text", str(e)) - error_response: Final = getattr(e, "response", None) + error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) @@ -1744,7 +1744,7 @@ class OpenAIFilesAPI(BaseLLM): client: OpenAI | AsyncOpenAI | None = None, _is_async: bool = False, ) -> OpenAI | AsyncOpenAI | None: - received_args: Final[Mapping[str, object]] = locals() + received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) openai_client: OpenAI | AsyncOpenAI | None = None if client is None: data: Final = {} @@ -2102,7 +2102,7 @@ class OpenAIBatchesAPI(BaseLLM): client: OpenAI | AsyncOpenAI | None = None, _is_async: bool = False, ) -> OpenAI | AsyncOpenAI | None: - received_args: Final[Mapping[str, object]] = locals() + received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) openai_client: OpenAI | AsyncOpenAI | None = None if client is None: data: Final = {} @@ -2314,7 +2314,7 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: OpenAI | None = None, ) -> OpenAI: - received_args: Final[Mapping[str, object]] = locals() + received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) if client is None: data: Final = {} for k, v in received_args.items(): @@ -2339,7 +2339,7 @@ class OpenAIAssistantsAPI(BaseLLM): organization: str | None, client: AsyncOpenAI | None = None, ) -> AsyncOpenAI: - received_args: Final[Mapping[str, object]] = locals() + received_args: Final[Mapping[str, object]] = types.MappingProxyType(locals()) if client is None: data: Final = {} for k, v in received_args.items(): diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b83ad964425..768d89b9a97 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -715,11 +715,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if item.get("type") == "reasoning": verbose_logger.debug("Handling reasoning item: %s", item) # Type assertion since we know it's a dict at this point - dict_item = cast(dict[str, Any], item) + dict_item = cast(dict[str, object], item) filtered_item = self._handle_reasoning_item(dict_item) else: # For other dict items, just pass through - filtered_item = cast(dict[str, Any], item) + filtered_item = cast(dict[str, object], item) validated_input.append(filtered_item) else: validated_input.append(item) @@ -727,7 +727,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): # Input is expected to be either str or List, no single BaseModel expected return input - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: """ Handle reasoning items specifically to filter out status=None using OpenAI's model. Issue: https://github.com/BerriAI/litellm/issues/13484 diff --git a/litellm/llms/openai/speech/guardrail_translation/handler.py b/litellm/llms/openai/speech/guardrail_translation/handler.py index 9e338e80632..3e84c1a7351 100644 --- a/litellm/llms/openai/speech/guardrail_translation/handler.py +++ b/litellm/llms/openai/speech/guardrail_translation/handler.py @@ -5,7 +5,8 @@ This module provides guardrail translation support for OpenAI's text-to-speech e The handler processes the 'input' text parameter (output is audio, so no text to guardrail). """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -15,6 +16,7 @@ if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.llms.openai import HttpxBinaryResponseContent + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class OpenAITextToSpeechHandler(BaseTranslation): @@ -33,7 +35,7 @@ class OpenAITextToSpeechHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process input text by applying guardrails. @@ -82,9 +84,9 @@ class OpenAITextToSpeechHandler(BaseTranslation): response: "HttpxBinaryResponseContent", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - user_api_key_dict: Any | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, request_data: dict | None = None, - ) -> Any: + ) -> "HttpxBinaryResponseContent": """ Process output - not applicable for text-to-speech. diff --git a/litellm/llms/openai/transcriptions/guardrail_translation/handler.py b/litellm/llms/openai/transcriptions/guardrail_translation/handler.py index 97fd1038d35..ef7a131d35e 100644 --- a/litellm/llms/openai/transcriptions/guardrail_translation/handler.py +++ b/litellm/llms/openai/transcriptions/guardrail_translation/handler.py @@ -5,7 +5,8 @@ This module provides guardrail translation support for OpenAI's audio transcript The handler processes the output transcribed text (input is audio, so no text to guardrail). """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation @@ -14,6 +15,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.utils import TranscriptionResponse @@ -33,7 +35,7 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - ) -> Any: + ) -> Mapping[str, object]: """ Process input - not applicable for audio transcription. @@ -57,9 +59,9 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation): response: "TranscriptionResponse", guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, - user_api_key_dict: Any | None = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, request_data: dict | None = None, - ) -> Any: + ) -> "TranscriptionResponse": """ Process output transcription by applying guardrails to transcribed text. diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index 8fb9dd1318e..8f96f688189 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from typing import Final from httpx import Headers, Response @@ -18,6 +19,10 @@ from litellm.types.utils import FileTypes, TranscriptionResponse from ..common_utils import OpenAIError +def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: + return headers.get(key, default) + + class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_complete_url( self, @@ -127,7 +132,7 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): try: raw_response_json: Final = raw_response.json() except json.JSONDecodeError: - content_type: Final = raw_response.headers.get("content-type", "").lower() + content_type: Final = _header_value(raw_response.headers, "content-type", "").lower() if "application/json" in content_type: raise return TranscriptionResponse(text=raw_response.text) diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index a27ea70fbdb..b0da53af8b3 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -7,7 +7,7 @@ to extract specific fields for guardrail processing. """ from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Final, Optional, Protocol, runtime_checkable from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import without_server_streaming_classification @@ -196,6 +196,21 @@ class PassThroughEndpointHandler(BaseTranslation): return response +@runtime_checkable +class _EventStreamContentTypeDetector(Protocol): + def is_event_stream_content_type(self, content_type: str) -> bool: ... + + +@runtime_checkable +class _EventStreamMediaTypeProvider(Protocol): + def event_stream_media_type(self) -> str: ... + + +@runtime_checkable +class _EventStreamEndpointChecker(Protocol): + def event_stream_endpoint_is_de_anonymizable(self, endpoint: str) -> bool: ... + + _PROVIDER_HANDLERS: dict[str, type[BaseTranslation]] = {} @@ -223,7 +238,7 @@ class LlmPassthroughRouteHandler(BaseTranslation): data: dict, guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, - ) -> Any: + ) -> object: provider: Final = data.get("custom_llm_provider") handler_cls: Final = _get_provider_handlers().get(provider or "") if handler_cls is None: @@ -265,18 +280,16 @@ class LlmPassthroughRouteHandler(BaseTranslation): @staticmethod def is_event_stream_response(provider: str | None, content_type: str) -> bool: handler_cls: Final = _get_provider_handlers().get(provider or "") - detector: Final = getattr(handler_cls, "is_event_stream_content_type", None) - if detector is None: + if not isinstance(handler_cls, _EventStreamContentTypeDetector): return False - return detector(content_type) + return handler_cls.is_event_stream_content_type(content_type) @staticmethod def event_stream_media_type(provider: str | None) -> str | None: handler_cls: Final = _get_provider_handlers().get(provider or "") - getter: Final = getattr(handler_cls, "event_stream_media_type", None) - if getter is None: + if not isinstance(handler_cls, _EventStreamMediaTypeProvider): return None - return getter() + return handler_cls.event_stream_media_type() @staticmethod def _resolve_event_stream_de_anonymizer(provider: str | None): @@ -286,10 +299,9 @@ class LlmPassthroughRouteHandler(BaseTranslation): @staticmethod def supports_event_stream_de_anonymization(provider: str | None, endpoint: str | None) -> bool: handler_cls: Final = _get_provider_handlers().get(provider or "") - endpoint_check: Final = getattr(handler_cls, "event_stream_endpoint_is_de_anonymizable", None) - if endpoint_check is None: + if not isinstance(handler_cls, _EventStreamEndpointChecker): return False - return endpoint_check(endpoint or "") + return handler_cls.event_stream_endpoint_is_de_anonymizable(endpoint or "") @staticmethod async def de_anonymize_event_stream( diff --git a/litellm/llms/petals/completion/handler.py b/litellm/llms/petals/completion/handler.py index 19010a6722a..425d3729d7c 100644 --- a/litellm/llms/petals/completion/handler.py +++ b/litellm/llms/petals/completion/handler.py @@ -1,5 +1,6 @@ import time -from collections.abc import Callable +import types +from collections.abc import Callable, Mapping from typing import Final import litellm @@ -32,7 +33,7 @@ def completion( client: HTTPHandler | AsyncHTTPHandler | None = None, ): ## Load Config - config: Final = litellm.PetalsConfig.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(litellm.PetalsConfig.get_config()) for k, v in config.items(): if ( k not in optional_params @@ -128,7 +129,8 @@ def completion( model_response.choices[0].message.content = output_text prompt_tokens: Final = len(encoding.encode(prompt)) - completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"].get("content"))) + response_message: Final = model_response.choices[0].message + completion_tokens: Final = len(encoding.encode(response_message.content)) model_response.created = int(time.time()) model_response.model = model diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index bb9b808df0a..7bd05c6e30b 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -1,5 +1,7 @@ import os import time +import types +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal from httpx import Headers, Response @@ -232,11 +234,11 @@ class PredibaseConfig(BaseConfig): except Exception: # Keep usage calculation non-blocking if token counting fails. pass - output_text: Final = model_response["choices"][0]["message"].get("content", "") + output_text: Final = model_response.choices[0].message.content if encoding is not None and output_text is not None and len(output_text) > 0: completion_tokens = 0 try: - completion_tokens = len(encoding.encode(model_response["choices"][0]["message"].get("content", ""))) + completion_tokens = len(encoding.encode(output_text)) except Exception: # Keep usage calculation non-blocking if encoding fails. pass @@ -285,7 +287,7 @@ class PredibaseConfig(BaseConfig): prompt = prompt_factory(model=model, messages=messages) request_optional_params: Final = {**optional_params} - config: Final = self.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(self.get_config()) for k, v in config.items(): if k not in request_optional_params: request_optional_params[k] = v diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 42e9eb2c3a6..62dfac43bd7 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -1,3 +1,4 @@ +import types from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final @@ -168,7 +169,7 @@ class ReplicateConfig(BaseConfig): headers: dict, ) -> dict: ## Load Config - config: Final = litellm.ReplicateConfig.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(litellm.ReplicateConfig.get_config()) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index b285200ef15..b89c59a0e5e 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -1,9 +1,11 @@ import json -from collections.abc import Callable +import types +from collections.abc import Callable, Mapping from copy import deepcopy from typing import Final, cast import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm @@ -36,6 +38,7 @@ from ..embedding.transformation import SagemakerEmbeddingConfig from .transformation import SagemakerConfig sagemaker_config: Final = SagemakerConfig() +_DECODED_JSON: Final = TypeAdapter(object) class _InvokeEndpointArgs(TypedDict): @@ -152,7 +155,7 @@ class SagemakerLLM(BaseAWSLLM): model_id: Final = optional_params.get("model_id", None) ## Load Config - config: Final = litellm.SagemakerConfig.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(litellm.SagemakerConfig.get_config()) for k, v in config.items(): if ( k not in inference_params @@ -573,7 +576,7 @@ class SagemakerLLM(BaseAWSLLM): inference_params.pop("stream", None) ## Load Config - config: Final = litellm.SagemakerConfig.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(litellm.SagemakerConfig.get_config()) for k, v in config.items(): if ( k not in inference_params @@ -623,7 +626,7 @@ class SagemakerLLM(BaseAWSLLM): status_code, error_message = client_error_details(e) raise SagemakerError(status_code=status_code, message=with_inference_component_hint(error_message)) - response = json.loads(response["Body"].read().decode("utf8")) + response = _DECODED_JSON.validate_python(json.loads(response["Body"].read().decode("utf8"))) ## LOGGING logging_obj.post_call( input=input, diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index e1bc496a82d..274f884de0b 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -6,6 +6,8 @@ In the Huggingface TGI format. import json import time +import types +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from httpx._models import Headers, Response @@ -51,7 +53,7 @@ class SagemakerConfig(BaseConfig): temperature: float | None = None, return_full_text: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @@ -239,8 +241,9 @@ class SagemakerConfig(BaseConfig): prompt_tokens: Final = token_counter( text=prompt, count_response_tokens=True ) # doesn't apply any default token count from openai's chat template + response_message: Final = model_response.choices[0].message completion_tokens: Final = token_counter( - text=model_response["choices"][0]["message"].get("content", ""), + text=response_message.content, count_response_tokens=True, ) diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index 43fa3236e88..f2b31d4f9b0 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -1,4 +1,5 @@ import warnings +from collections.abc import Mapping from enum import Enum from typing import Final, Literal @@ -6,13 +7,24 @@ from pydantic import ( Field, SerializationInfo, SerializerFunctionWrapHandler, + TypeAdapter, field_validator, model_serializer, model_validator, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.base import LiteLLMBaseModel +_SERIALIZED: Final = TypeAdapter(object) + + +def _serialized_fields(serialized: object) -> Mapping[str, object]: + fields: Final = as_str_mapping(serialized) + if fields is None: + raise TypeError(f"model serializer produced {type(serialized).__name__}, expected a mapping") + return fields + class CacheControl(LiteLLMBaseModel): type: Literal["ephemeral"] @@ -22,10 +34,9 @@ class CacheControl(LiteLLMBaseModel): def _serialize( self, handler: SerializerFunctionWrapHandler, info: SerializationInfo ) -> dict: # mutable-ok: pydantic serializer contract requires bare dict return - result = handler(self) - if result.get("ttl") is None: - result.pop("ttl", None) - return result + fields: Final = _serialized_fields(_SERIALIZED.validate_python(handler(self))) + serialized: Final = {k: v for k, v in fields.items() if k != "ttl" or v is not None} + return serialized class TextContent(LiteLLMBaseModel): @@ -37,10 +48,9 @@ class TextContent(LiteLLMBaseModel): def _serialize( self, handler: SerializerFunctionWrapHandler, info: SerializationInfo ) -> dict: # mutable-ok: pydantic serializer contract requires bare dict return - result = handler(self) - if result.get("cache_control") is None: - result.pop("cache_control", None) - return result + fields: Final = _serialized_fields(_SERIALIZED.validate_python(handler(self))) + serialized: Final = {k: v for k, v in fields.items() if k != "cache_control" or v is not None} + return serialized class ImageURLContent(LiteLLMBaseModel): @@ -57,10 +67,9 @@ class ImageContent(LiteLLMBaseModel): def _serialize( self, handler: SerializerFunctionWrapHandler, info: SerializationInfo ) -> dict: # mutable-ok: pydantic serializer contract requires bare dict return - result = handler(self) - if result.get("cache_control") is None: - result.pop("cache_control", None) - return result + fields: Final = _serialized_fields(_SERIALIZED.validate_python(handler(self))) + serialized: Final = {k: v for k, v in fields.items() if k != "cache_control" or v is not None} + return serialized class FunctionObj(LiteLLMBaseModel): diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 704e81de16a..a82dfe72322 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -2,6 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion` """ +import types from collections.abc import AsyncIterator, Iterator, Mapping from functools import cached_property from typing import TYPE_CHECKING, Any, Final, Union @@ -141,7 +142,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): tools: list | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index 452dcc15c4c..b1618f38623 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -11,6 +11,7 @@ from threading import Lock from typing import Any, Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -29,6 +30,8 @@ SERVICE_KEY_ENV_VAR: Final = "AICORE_SERVICE_KEY" DEFAULT_HOME_PATH: Final = os.path.join(os.path.expanduser("~"), ".aicore") +_DECODED_JSON: Final = TypeAdapter(object) + def _get_home() -> str: return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH) @@ -39,7 +42,7 @@ def _get_nested(d: object, path: Sequence[str]) -> object: if isinstance(cur, str): # This shouldn't happen if service keys are pre-parsed correctly try: - cur = json.loads(cur) + cur = _DECODED_JSON.validate_python(json.loads(cur)) except json.JSONDecodeError: verbose_logger.warning("SAP service key or VCAP service is a string but not valid JSON.") return None diff --git a/litellm/llms/vertex_ai/agent_engine/sse_iterator.py b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py index a0b2b725f36..1d1901dd97c 100644 --- a/litellm/llms/vertex_ai/agent_engine/sse_iterator.py +++ b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py @@ -4,7 +4,7 @@ SSE Stream Iterator for Vertex AI Agent Engine. Handles Server-Sent Events (SSE) streaming responses from Vertex AI Reasoning Engines. """ -from typing import Any, Final +from typing import Final from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import ChatCompletionUsageBlock @@ -24,7 +24,7 @@ class VertexAgentEngineResponseIterator(BaseModelResponseIterator): We just need to implement chunk_parser to parse Vertex Agent Engine response format. """ - def __init__(self, streaming_response: Any, sync_stream: bool) -> None: + def __init__(self, streaming_response: object, sync_stream: bool) -> None: super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) def chunk_parser(self, chunk: dict) -> GenericStreamingChunk | ModelResponseStream: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 10e89165b73..9f592398f74 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -14,6 +14,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Optional, Union import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -42,6 +43,12 @@ else: AsyncHTTPHandler = Any CustomStreamWrapper = Any +_DECODED_JSON: Final = TypeAdapter(object) + + +def _content_type(headers: Mapping[str, str]) -> str: + return headers.get("content-type", "") + class VertexAgentEngineError(BaseLLMException): """Exception for Vertex Agent Engine errors.""" @@ -296,7 +303,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): We need to collect all the chunks and extract the final response. """ try: - content_type: Final = raw_response.headers.get("content-type", "").lower() + content_type: Final = _content_type(raw_response.headers).lower() verbose_logger.debug("Vertex Agent Engine response Content-Type: %s", content_type) # Parse the SSE response @@ -311,7 +318,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): continue try: - data = json.loads(line) + data = _DECODED_JSON.validate_python(json.loads(line)) if isinstance(data, dict): text = self._extract_text_from_response(data) if text: diff --git a/litellm/llms/vertex_ai/count_tokens/handler.py b/litellm/llms/vertex_ai/count_tokens/handler.py index 03970099f2e..f8d37350ee9 100644 --- a/litellm/llms/vertex_ai/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/count_tokens/handler.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Final +from typing import Final from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -23,10 +23,10 @@ class VertexAITokenCounter(GoogleAIStudioTokenCounter, VertexBase): self, api_base: str | None = None, api_key: str | None = None, - headers: dict[str, Any] | None = None, + headers: dict[str, object] | None = None, model: str = "", - litellm_params: dict[str, Any] | None = None, - ) -> tuple[dict[str, Any], str]: + litellm_params: dict[str, object] | None = None, + ) -> tuple[dict[str, object], str]: """ Returns a Tuple of headers and url for the Vertex AI countTokens endpoint. """ diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index c392baa968b..396acd270fa 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -853,7 +853,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): def __init__( self, openai_file_content: FileTypes, - map_openai_to_vertex_params: Callable[[dict[str, Any]], dict[str, object]], + map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index e1acea53192..40564d536a7 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -183,7 +183,9 @@ class VertexFineTuningAPI(VertexLLM): f"Error creating fine tuning job. Status code: {response.status_code}. Response: {response.text}" ) - verbose_logger.debug("got response from creating fine tuning job: %s", response.json()) + verbose_logger.debug( + "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) + ) vertex_response: Final = ResponseTuningJob( **response.json(), @@ -267,7 +269,9 @@ class VertexFineTuningAPI(VertexLLM): f"Error creating fine tuning job. Status code: {response.status_code}. Response: {response.text}" ) - verbose_logger.debug("got response from creating fine tuning job: %s", response.json()) + verbose_logger.debug( + "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) + ) vertex_response: Final = ResponseTuningJob( **response.json(), ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 15e5e440b91..909ca526233 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from urllib.parse import quote import httpx -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._logging import verbose_logger @@ -46,6 +46,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.llms.vertex_ai import * from litellm.types.llms.vertex_ai import ( + VERTEX_CREDENTIALS_TYPES, GenerationConfig, PartType, RequestBody, @@ -64,13 +65,14 @@ from ..common_utils import ( get_supports_system_message, ) -# Typed as Any to avoid introducing a module-load-time cyclic import to -# vertex_llm_base. The instance is lazily constructed by _get_vertex_base() -# the first time GCS metadata needs to be fetched. -_GCS_METADATA_VERTEX_BASE: object | None = None +# VertexBase is imported only for type checking to avoid introducing a +# module-load-time cyclic import to vertex_llm_base. The instance is lazily +# constructed by _get_vertex_base() the first time GCS metadata needs to be fetched. +_GCS_METADATA_VERTEX_BASE: "VertexBase | None" = None # Shared sync client for GCS JSON API metadata reads so proxy/SSL settings # from litellm's HTTP stack apply (see Greptile review on PR #27278). _GCS_METADATA_HTTP_HANDLER: HTTPHandler | None = None +_DECODED_JSON: Final = TypeAdapter(object) _GEMINI_MIME_TYPE_ALIASES: Final[dict[str, str]] = { "image/jpg": "image/jpeg", } @@ -86,7 +88,7 @@ def _apply_gemini_mime_type_aliases(mime_type: str) -> str: return _GEMINI_MIME_TYPE_ALIASES.get(normalized, normalized) -def _get_vertex_base() -> Any: +def _get_vertex_base() -> "VertexBase": """Lazily return the shared VertexBase instance to avoid a module-load-time cyclic import.""" global _GCS_METADATA_VERTEX_BASE if _GCS_METADATA_VERTEX_BASE is None: @@ -106,6 +108,8 @@ def _get_gcs_metadata_http_handler() -> HTTPHandler: if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from ..vertex_llm_base import VertexBase + LiteLLMLoggingObj = _LiteLLMLoggingObj else: LiteLLMLoggingObj = Any @@ -323,7 +327,7 @@ def _openai_messages_may_need_sync_gcs_metadata_fetch( def _get_gcs_object_content_type( image_url: str, vertex_project: str | None = None, - vertex_credentials: object = None, + vertex_credentials: VERTEX_CREDENTIALS_TYPES | None = None, ) -> str | None: """ Resolve content type from GCS object metadata. @@ -400,7 +404,7 @@ def _get_gcs_object_content_type( return None try: - payload: Final = response.json() + payload: Final = _DECODED_JSON.validate_python(response.json()) except ValueError as e: if explicit_vertex_auth_provided: raise litellm.BadRequestError( @@ -476,7 +480,7 @@ def _process_gemini_media( model: str | None = None, video_metadata: Mapping[str, object] | None = None, vertex_project: str | None = None, - vertex_credentials: object = None, + vertex_credentials: VERTEX_CREDENTIALS_TYPES | None = None, ) -> PartType: """ Given a media URL (image, audio, or video), return the appropriate PartType for Gemini diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 109dddf44ed..be808e05202 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args import httpx from pydantic import JsonValue @@ -116,6 +116,20 @@ else: SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "none", "disable") + +def _response_header(headers: Mapping[str, str], name: str) -> str | None: + return headers.get(name) + + +class _HiddenParamsSink(Protocol): + def __setitem__(self, key: str, value: object, /) -> None: ... + + +class _StreamMetadataTarget(Protocol): + @property + def hidden_params(self) -> _HiddenParamsSink: ... + + # Gemini prebuilt voices per https://ai.google.dev/gemini-api/docs/speech-generation (2026-10-01), by nearest character; # nova is left out since Gemini 3.x TTS models accept it as given (live check 2026-10-08) OPENAI_TO_GEMINI_TTS_VOICES: Final[Mapping[str, str]] = MappingProxyType( @@ -261,7 +275,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): presence_penalty: float | None = None, seed: int | None = None, ) -> None: - locals_: Final[Mapping[str, object]] = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @@ -2216,7 +2230,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _set_stream_metadata_on_response( - model_response: Any, + model_response: _StreamMetadataTarget, grounding_metadata: Sequence[Mapping[str, object]], url_context_metadata: Sequence[Mapping[str, object]], safety_ratings: Sequence[Mapping[str, object]], @@ -2231,11 +2245,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) setattr(model_response, "vertex_ai_safety_results", safety_ratings) if safety_ratings: - model_response._hidden_params["vertex_ai_safety_ratings"] = safety_ratings - model_response._hidden_params["vertex_ai_safety_results"] = safety_ratings + model_response.hidden_params["vertex_ai_safety_ratings"] = safety_ratings + model_response.hidden_params["vertex_ai_safety_results"] = safety_ratings setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) if citation_metadata: - model_response._hidden_params["vertex_ai_citation_metadata"] = citation_metadata + model_response.hidden_params["vertex_ai_citation_metadata"] = citation_metadata @classmethod def set_stream_metadata_on_response( @@ -2406,11 +2420,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) if audio_response is not None: - cast(dict[str, Any], chat_completion_message)["audio"] = audio_response + cast(dict[str, object], chat_completion_message)["audio"] = audio_response chat_completion_message["content"] = None # OpenAI spec if image_response is not None: # Handle image response - combine with text content into structured format - cast(dict[str, Any], chat_completion_message)["images"] = image_response + cast(dict[str, object], chat_completion_message)["images"] = image_response if content is not None: chat_completion_message["content"] = content @@ -2654,7 +2668,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ## ADD SERVICE TIER ## if getattr(raw_response, "headers", None): - service_tier: Final[str | None] = raw_response.headers.get("x-gemini-service-tier") + service_tier: Final = _response_header(raw_response.headers, "x-gemini-service-tier") if service_tier: if service_tier.lower() == "standard": setattr(model_response, "service_tier", "default") diff --git a/litellm/llms/vertex_ai/google_genai/transformation.py b/litellm/llms/vertex_ai/google_genai/transformation.py index 8c22ae06af0..563a87b862d 100644 --- a/litellm/llms/vertex_ai/google_genai/transformation.py +++ b/litellm/llms/vertex_ai/google_genai/transformation.py @@ -2,7 +2,7 @@ Transformation for Calling Google models in their native format. """ -from typing import Any, Final, Literal +from typing import Final, Literal from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig from litellm.types.router import GenericLiteLLMParams @@ -70,10 +70,10 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig): def transform_generate_content_request( self, model: str, - contents: Any, - tools: Any | None, + contents: object, + tools: object | None, generate_content_config_dict: dict, - system_instruction: Any | None = None, + system_instruction: object | None = None, ) -> dict: """ Transform the generate content request for Vertex AI. diff --git a/litellm/llms/vertex_ai/rag_engine/ingestion.py b/litellm/llms/vertex_ai/rag_engine/ingestion.py index c10bac595b6..5a4089bdf41 100644 --- a/litellm/llms/vertex_ai/rag_engine/ingestion.py +++ b/litellm/llms/vertex_ai/rag_engine/ingestion.py @@ -94,7 +94,7 @@ class VertexAIRAGIngestion(BaseRAGIngestion): ) # Import settings - self.wait_for_import = self.vector_store_config.get("wait_for_import", True) + self.wait_for_import: object = self.vector_store_config.get("wait_for_import", True) self.import_timeout = _get_int(self.vector_store_config.get("import_timeout"), 600) # Validate required config diff --git a/litellm/llms/vertex_ai/vertex_ai_non_gemini.py b/litellm/llms/vertex_ai/vertex_ai_non_gemini.py index 9d960845fd3..cb3fc6304e6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_non_gemini.py +++ b/litellm/llms/vertex_ai/vertex_ai_non_gemini.py @@ -1,6 +1,7 @@ import os import time -from collections.abc import Callable +import types +from collections.abc import Callable, Mapping from typing import Any, Final, cast import httpx @@ -161,7 +162,7 @@ def completion( ) ## Load Config - config: Final = litellm.VertexAIConfig.get_config() + config: Final[Mapping[str, object]] = types.MappingProxyType(litellm.VertexAIConfig.get_config()) for k, v in config.items(): if k not in optional_params: optional_params[k] = v @@ -413,7 +414,7 @@ def completion( completion_tokens = response_obj.usage_metadata.candidates_token_count else: prompt_tokens = len(encoding.encode(prompt)) - completion_tokens = len(encoding.encode(model_response["choices"][0]["message"].get("content", ""))) + completion_tokens = len(encoding.encode(model_response.choices[0].message.content)) usage = Usage( prompt_tokens=prompt_tokens, @@ -559,7 +560,7 @@ async def async_completion( completion_tokens = response_obj.usage_metadata.candidates_token_count else: prompt_tokens = len(encoding.encode(prompt)) - completion_tokens = len(encoding.encode(model_response["choices"][0]["message"].get("content", ""))) + completion_tokens = len(encoding.encode(model_response.choices[0].message.content)) # set usage usage = Usage( diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index e72780fd943..d11be440172 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -1,5 +1,5 @@ import types -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Final import httpx @@ -45,7 +45,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): self, max_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key == "max_tokens" and value is None: value = self.max_tokens @@ -54,9 +54,10 @@ class VertexAILlama3Config(OpenAIGPTConfig): @classmethod def get_config(cls): + config_attrs: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in config_attrs.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index 7c626c66e9d..b298c9996f7 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -3,6 +3,7 @@ Volcengine Embedding Transformation Transforms OpenAI embedding requests to Volcengine format """ +import types from collections.abc import Mapping from typing import Final @@ -27,7 +28,7 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): self, encoding_format: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py index 2e1e8fac643..1b870c7f639 100644 --- a/litellm/llms/watsonx/embed/transformation.py +++ b/litellm/llms/watsonx/embed/transformation.py @@ -3,10 +3,11 @@ Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route. """ from collections.abc import Iterable, Mapping -from typing import Final +from typing import Final, Protocol import httpx from pydantic import ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.embedding.transformation import ( BaseEmbeddingConfig, @@ -22,6 +23,20 @@ _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=Config _TOKEN_COUNT: Final = TypeAdapter(int) +class _WatsonXEmbeddingBody(TypedDict, total=False): + model_id: ReadOnly[str | None] + results: ReadOnly[object] + input_token_count: ReadOnly[object] + + +class _WatsonXEmbeddingHTTPResponse(Protocol): + def json(self) -> _WatsonXEmbeddingBody: ... + + +def _embedding_body(response: _WatsonXEmbeddingHTTPResponse) -> _WatsonXEmbeddingBody: + return response.json() + + class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): def get_supported_openai_params(self, model: str) -> list: return [] @@ -97,7 +112,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): logging_obj.post_call( original_response=raw_response.text, ) - json_resp: Final = raw_response.json() + json_resp: Final = _embedding_body(raw_response) if model_response is None: model_response = EmbeddingResponse(model=json_resp.get("model_id", None)) results: Final = _JSON_OBJECTS.validate_python(json_resp.get("results", [])) diff --git a/litellm/main.py b/litellm/main.py index f4c06565e85..b93c61ead16 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6834,7 +6834,7 @@ def embedding( response = huggingface_embed.embedding( model=model, input=input, - encoding=sys.modules[__name__].encoding, + encoding=_load_module_encoding(), api_key=api_key, api_base=api_base, logging_obj=logging, @@ -9064,9 +9064,9 @@ _encoding_cache: Tokenizer | None = None def _load_module_encoding() -> Tokenizer: - import sys + from litellm import main as litellm_main - return sys.modules[__name__].encoding + return litellm_main.encoding def _get_encoding() -> Tokenizer: @@ -9089,8 +9089,6 @@ def __getattr__(name: str) -> Tokenizer: if name == "encoding": _encoding: Final = _load_default_encoding() # Cache it in the module's __dict__ for subsequent accesses - import sys - sys.modules[__name__].__dict__["encoding"] = _encoding global _encoding_cache _encoding_cache = _encoding diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index ff9eb8fe79a..97188217f63 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -9,7 +9,7 @@ import contextvars from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Coroutine, Generator, Iterator from functools import partial from types import TracebackType -from typing import Any, Final, cast +from typing import Final, cast import httpx from httpx._types import CookieTypes, QueryParamTypes, RequestContent, RequestFiles @@ -104,7 +104,7 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]): def headers(self, value: httpx.Headers) -> None: self._headers = value - def __await__(self) -> Iterator[Any]: + def __await__(self) -> Generator[object, None, AsyncPassthroughStreamingResponse]: async def _init(): if not self._initialized: self._response = await self._response_coro diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 43537d48f5e..63f73f567eb 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -41,6 +41,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credentials import ( is_session_bearer_shaped, ) +from litellm.proxy._experimental.mcp_server.utils import asgi_scope_fields from litellm.proxy._types import ( UI_TEAM_ID, LiteLLM_AccessGroupTable, @@ -1705,7 +1706,7 @@ class MCPRequestHandler: ``Authorization`` headers is malformed for bearer auth anyway (RFC 9110: not a comma-combinable field), so fail closed with a 400. """ - raw_headers: Final = scope.get("headers", []) + raw_headers: Final = asgi_scope_fields(scope).get("headers", []) MCPRequestHandler._reject_duplicate_authorization(raw_headers) try: # ASGI headers are list of [name: bytes, value: bytes] pairs diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index b9a1a73549a..a209bd88b9e 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -297,6 +297,7 @@ def decode_state_hash(encrypted_state: str) -> dict: _MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_" +_DECODED_JSON: Final = TypeAdapter(object) class _ManagedAuthorizationCode(LiteLLMBaseModel): @@ -2823,7 +2824,7 @@ async def fetch_upstream_oauth_protected_resource( continue if response.status_code == 200: try: - payload = response.json() + payload = _DECODED_JSON.validate_python(response.json()) except Exception as exc: verbose_logger.warning( "MCP OAuth metadata at %s returned 200 but JSON " diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 906e2c03dc5..968a3240191 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -284,6 +284,11 @@ _USER_ENV_VARS_CACHE_MAX_SIZE: Final = 4096 # cap to prevent unbounded growth _ListedToolsByCaller: TypeAlias = Mapping[str | None, Mapping[str, MCPTool]] _LISTED_TOOLS_CALLERS_PER_SERVER: Final = 256 +_UNTYPED_VALUE: Final = TypeAdapter(object) + + +def _www_authenticate_header(headers: Mapping[str, str]) -> str | None: + return headers.get("WWW-Authenticate") or headers.get("www-authenticate") @dataclass(frozen=True, slots=True) @@ -1708,7 +1713,7 @@ def _deserialize_json_list(data: object) -> list[dict[str, Any]] | None: return None if isinstance(data, str): try: - parsed: Final[object] = json.loads(data) + parsed: Final = _UNTYPED_VALUE.validate_python(json.loads(data)) except (json.JSONDecodeError, TypeError): return None data = parsed @@ -2910,7 +2915,7 @@ class MCPServerManager: verbose_logger.debug("Processing %s paths from OpenAPI spec", len(paths)) for path, path_item in paths.items(): - for method in ["get", "post", "put", "delete", "patch"]: + for method in ("get", "post", "put", "delete", "patch"): if method not in path_item: continue @@ -3512,7 +3517,9 @@ class MCPServerManager: byom_cache_key: Final = self.get_byom_submitted_servers_cache_key(submitter_user_id) submitted_server_ids: list[str] | None = None try: - cached_submitted_server_ids: Final = await user_api_key_cache.async_get_cache(key=byom_cache_key) + cached_submitted_server_ids: Final = _UNTYPED_VALUE.validate_python( + await user_api_key_cache.async_get_cache(key=byom_cache_key) + ) if cached_submitted_server_ids is not None: submitted_server_ids = cast(list[str], cached_submitted_server_ids) except Exception as e: # noqa: BLE001 @@ -5165,7 +5172,7 @@ class MCPServerManager: header_value: str | None = None if exc.response is not None: - header_value = exc.response.headers.get("WWW-Authenticate") or exc.response.headers.get("www-authenticate") + header_value = _www_authenticate_header(exc.response.headers) status_attempt: Final = ( f"GET {origin}: HTTP {exc.response.status_code}" if exc.response is not None diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index d0133739787..f0353f309a9 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -9,7 +9,7 @@ import os import re from collections.abc import Mapping, Sequence from pathlib import PurePosixPath -from typing import Any, Final, TypedDict +from typing import Final, Literal, Protocol, TypeAlias, TypedDict, overload from urllib.parse import quote import httpx @@ -91,16 +91,40 @@ class _OpenAPIOperation(TypedDict, total=False): requestBody: _OpenAPIRequestBody -class _OpenAPIPathItem(TypedDict, total=False): - summary: str - description: str - parameters: Sequence[_OpenAPIParameter] +_OpenAPIMethod: TypeAlias = Literal["get", "post", "put", "delete", "patch"] + + +class _OpenAPIPathItem(Protocol): + def __contains__(self, key: object, /) -> bool: ... + + def __getitem__(self, key: _OpenAPIMethod, /) -> _OpenAPIOperation: ... + + @overload + def get(self, key: _OpenAPIMethod, /) -> _OpenAPIOperation | None: ... + + @overload + def get( + self, key: Literal["parameters"], default: Sequence[_OpenAPIParameter], / + ) -> Sequence[_OpenAPIParameter]: ... class _OpenAPIComponents(TypedDict, total=False): parameters: Mapping[str, _OpenAPIParameter] +class _OpenAPIServer(TypedDict): + url: ReadOnly[str] + + +class _OpenAPISpec(TypedDict, total=False): + paths: ReadOnly[Mapping[str, _OpenAPIPathItem]] + components: ReadOnly[_OpenAPIComponents] + servers: ReadOnly[Sequence[_OpenAPIServer]] + host: ReadOnly[str] + schemes: ReadOnly[Sequence[str]] + basePath: ReadOnly[str] + + # Store the base URL and headers globally BASE_URL: Final = "" HEADERS: Final[dict[str, str]] = {} @@ -157,7 +181,7 @@ def _sanitize_path_parameter_value(param_value: object, param_name: str) -> str: return quote(value_str, safe="") -def load_openapi_spec(filepath: str) -> dict[str, object]: +def load_openapi_spec(filepath: str) -> _OpenAPISpec: """ Sync wrapper. For URL specs, use the shared/custom MCP httpx client. """ @@ -175,7 +199,7 @@ def load_openapi_spec(filepath: str) -> dict[str, object]: return asyncio.run(load_openapi_spec_async(filepath)) -async def load_openapi_spec_async(filepath: str, *, max_bytes: int | None = None) -> dict[str, Any]: +async def load_openapi_spec_async(filepath: str, *, max_bytes: int | None = None) -> _OpenAPISpec: if filepath.startswith("http://") or filepath.startswith("https://"): client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) r: Final[httpx.Response] = ( @@ -194,7 +218,7 @@ async def load_openapi_spec_async(filepath: str, *, max_bytes: int | None = None return json.load(f) -def get_base_url(spec: Mapping[str, Any], spec_path: str | None = None) -> str: +def get_base_url(spec: _OpenAPISpec, spec_path: str | None = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: @@ -604,13 +628,13 @@ def create_tool_function( return tool_function -def register_tools_from_openapi(spec: Mapping[str, Any], base_url: str) -> None: +def register_tools_from_openapi(spec: _OpenAPISpec, base_url: str) -> None: """Register MCP tools from OpenAPI specification.""" - paths: Final[Mapping[str, Mapping[str, _OpenAPIOperation]]] = spec.get("paths", {}) + paths: Final = spec.get("paths", {}) used_names: Final = set() for path, path_item in paths.items(): - for method in ["get", "post", "put", "delete", "patch"]: + for method in ("get", "post", "put", "delete", "patch"): if method in path_item: operation = path_item[method] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index b0e6d08e649..d1302a2d505 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -30,6 +30,7 @@ from litellm.constants import ( MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH, MCP_PEEKED_BODY_SCOPE_KEY, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -89,6 +90,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, LITELLM_MCP_SERVER_VERSION, + asgi_scope_fields, merge_mcp_headers, ) from litellm.proxy._types import ( @@ -130,6 +132,7 @@ _MCP_ROUTING_PEEK_MAX_BYTES: Final = 4096 _MCP_TRANSPORT_SPAN_SCOPE_KEY: Final = "litellm_otel_transport_span" _MCP_DESTINATIONS_SCOPE_KEY: Final = "litellm_otel_request_destinations" _MCP_PROTOCOL_VERSION_HEADER: Final = b"mcp-protocol-version" +_DECODED_JSON: Final = TypeAdapter(object) def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: @@ -139,6 +142,14 @@ def reject_disallowed_mcp_origin(request: StarletteRequest) -> None: raise HTTPException(status_code=403, detail="Invalid Origin header") +def _scope_path(scope: Scope) -> str: + return asgi_scope_fields(scope).get("path", "") + + +def _scope_headers(scope: Scope) -> Sequence[tuple[bytes, bytes]]: + return asgi_scope_fields(scope).get("headers", []) + + def unsupported_protocol_version(scope: Scope) -> str | None: """Admit configured HTTP revisions while keeping SSE on the legacy protocol.""" from litellm.proxy._experimental.mcp_server.capabilities import configured_versions @@ -148,7 +159,7 @@ def unsupported_protocol_version(scope: Scope) -> str | None: raw.decode("latin-1").strip() for key, raw in headers if key.lower() == _MCP_PROTOCOL_VERSION_HEADER ) for value in values: - if value == "2026-07-28" and scope.get("path", "").rstrip("/").endswith("/sse"): + if value == "2026-07-28" and _scope_path(scope).rstrip("/").endswith("/sse"): return value if value and value not in configured_versions(): return value @@ -1377,8 +1388,8 @@ if MCP_AVAILABLE: if not body: return False try: - data: Final = json.loads(body) - return isinstance(data, dict) and data.get("method") == "initialize" + data: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body))) + return data is not None and data.get("method") == "initialize" except (json.JSONDecodeError, UnicodeDecodeError, TypeError): return False @@ -1678,7 +1689,7 @@ if MCP_AVAILABLE: return False # --- Session not in this worker's memory --- - method: Final = scope.get("method", "").upper() + method: Final = asgi_scope_fields(scope).get("method", "").upper() if method == "DELETE": _remove_stateful_session_tracking(_session_id) @@ -2025,7 +2036,7 @@ if MCP_AVAILABLE: def _get_authorization_header_from_scope(scope: Scope) -> str | None: """First ``Authorization`` header value in the ASGI scope, or None.""" - scope_headers: Final[Sequence[tuple[bytes, bytes]]] = scope.get("headers", []) + scope_headers: Final = _scope_headers(scope) for key, value in scope_headers: if key.lower() == b"authorization": return value.decode("latin-1") @@ -2045,7 +2056,7 @@ if MCP_AVAILABLE: ``MCPRequestHandler.process_mcp_request``), and forwarding it upstream would leak the proxy key to a third-party MCP server. """ - scope_headers: Final[Sequence[tuple[bytes, bytes]]] = scope.get("headers", []) + scope_headers: Final = _scope_headers(scope) has_litellm_key_header: Final = any(key.lower() == b"x-litellm-api-key" for key, _ in scope_headers) if not has_litellm_key_header: return None @@ -2285,7 +2296,7 @@ if MCP_AVAILABLE: }, )(scope, receive, send) return - path: Final[str] = scope.get("path", "") + path: Final = _scope_path(scope) peek: Final = _LazyPeekedBody(receive) if scope.get("method") == "POST" else None if peek is not None: receive = peek.receive # rebind-ok: replay peeked ASGI messages to the downstream handler @@ -2320,7 +2331,7 @@ if MCP_AVAILABLE: ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. - scope_headers: Final[Sequence[tuple[bytes, bytes]]] = scope.get("headers", []) + scope_headers: Final = _scope_headers(scope) scope["headers"] = [(k, v) for k, v in scope_headers if k.lower() != b"x-mcp-toolset-id"] # Apply toolset scope if set server-side via ContextVar (set by @@ -2503,9 +2514,9 @@ if MCP_AVAILABLE: request_method: Final = (scope.get("method") or "").upper() if body and request_method == "POST": try: - _peeked: Final = json.loads(body) + _peeked: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(body))) if ( - isinstance(_peeked, dict) + _peeked is not None and _peeked.get("jsonrpc") == "2.0" and "id" in _peeked and "method" not in _peeked @@ -2693,7 +2704,7 @@ if MCP_AVAILABLE: ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. - scope_headers: Final[Sequence[tuple[bytes, bytes]]] = scope.get("headers", []) + scope_headers: Final = _scope_headers(scope) scope["headers"] = [(k, v) for k, v in scope_headers if k.lower() != b"x-mcp-toolset-id"] # Apply toolset scope if set server-side via ContextVar so the diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 63dfd090f58..b2e3eae8e59 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -10,7 +10,7 @@ import re import typing from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from collections.abc import Set as AbstractSet -from typing import Any, Final, Protocol +from typing import Any, Final, Literal, Protocol, overload from urllib.parse import quote from litellm._logging import verbose_logger @@ -23,6 +23,20 @@ MCP_SERVERS_TARGET: Final = "mcp_servers" MCP_OAUTH_TOKENS_TARGET: Final = "mcp_oauth_tokens" +class ASGIScopeFields(Protocol): + @overload + def get( + self, key: Literal["headers"], default: Sequence[tuple[bytes, bytes]], / + ) -> Sequence[tuple[bytes, bytes]]: ... + + @overload + def get(self, key: Literal["path", "method"], default: str, /) -> str: ... + + +def asgi_scope_fields(scope: ASGIScopeFields) -> ASGIScopeFields: + return scope + + class _McpServerLike(Protocol): @property def server_id(self) -> str: ... diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 1da42fae6b6..a909a7a993a 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -368,7 +368,7 @@ class LazyFeatureMiddleware: # resolves SERVER_ROOT_PATHS prefixes there per request. The # `+ "/"` boundary prevents false-positive matches (e.g. /apiv2 # against root /api); a pre-stripped path is left alone. - root_path: Final = str(scope.get("root_path", "")).rstrip("/") or self._root_path + root_path: Final = str(_scope_value(scope, "root_path")).rstrip("/") or self._root_path if root_path and path.startswith(root_path + "/"): path = path[len(root_path) :] # rebind-ok: local strip after the boundary check above for feat in self._features: @@ -380,6 +380,10 @@ class LazyFeatureMiddleware: await self.app(scope, receive, send) +def _scope_value(scope: Mapping[str, object], key: str) -> object: + return scope.get(key, "") + + def _lazy_slots(app: "FastAPI") -> Mapping[str, BaseRoute | None]: return app.state.lazy_slots if hasattr(app.state, "lazy_slots") else MappingProxyType({}) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 6e972047712..98bb0985f4a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -245,6 +245,7 @@ def _allow_model_level_clientside_configurable_parameters( reject_server_owned_wif_params = _reject_server_owned_wif_params _CREDENTIAL_VALUES: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def reject_federated_credential_reference(body: Mapping[str, object]) -> None: @@ -697,7 +698,7 @@ def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None: if isinstance(value, str): from litellm.litellm_core_utils.safe_json_loads import safe_json_loads - parsed: Final = safe_json_loads(value) + parsed: Final = _DECODED_JSON.validate_python(safe_json_loads(value)) if isinstance(parsed, dict): return parsed return None @@ -1733,7 +1734,7 @@ def _coerce_user_id_to_str(value: object) -> str | None: # behind the flag preserves backwards compatibility for deployments # that intentionally pass JSON-encoded user identifiers. if litellm.validate_end_user_id_in_db and stripped[:1] in ("{", "["): - parsed: Final[object] = safe_json_loads(stripped) + parsed: Final[object] = _DECODED_JSON.validate_python(safe_json_loads(stripped)) if isinstance(parsed, (dict, list)): return None return stripped @@ -1799,7 +1800,7 @@ def get_end_user_id_from_request_body( if isinstance(value, dict): return value if isinstance(value, str): - parsed: Final = safe_json_loads(value) + parsed: Final = _DECODED_JSON.validate_python(safe_json_loads(value)) return parsed if isinstance(parsed, dict) else {} return {} diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 3283a080c14..2421eb10c41 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -25,6 +25,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from fastapi import HTTPException, status from jwt.api_jwk import PyJWK +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target @@ -108,6 +109,7 @@ JWKS_UNREACHABLE_BACKOFF_SECONDS: Final = 30 STALE_CACHE_KEY_PREFIX: Final = "litellm_stale_" STALE_WRITTEN_AT_CACHE_KEY_PREFIX: Final = "litellm_stale_written_at_" UNREACHABLE_CACHE_KEY_PREFIX: Final = "litellm_jwks_unreachable_" +_CACHED_VALUE: Final = TypeAdapter(object) _CachedValueT = TypeVar("_CachedValueT", bound=JWKKeyValue | str) @@ -785,12 +787,12 @@ class JWTHandler: @with_service_target(AUTH_OBJECTS_TARGET) async def _get_cached_value(self, cache_key: str) -> _CachedValueT | None: - cached: Final = await self.user_api_key_cache.async_get_cache(cache_key) + cached: Final = _CACHED_VALUE.validate_python(await self.user_api_key_cache.async_get_cache(cache_key)) return cast("_CachedValueT | None", cached) # cast-ok: cache reads are untyped @with_service_target(AUTH_OBJECTS_TARGET) async def _get_cached_timestamp(self, cache_key: str) -> float | None: - cached: Final = await self.user_api_key_cache.async_get_cache(cache_key) + cached: Final = _CACHED_VALUE.validate_python(await self.user_api_key_cache.async_get_cache(cache_key)) # A JSON round-trip through Redis hands a whole-number epoch back as an int. return float(cached) if isinstance(cached, (int, float)) else None diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 579d020f936..5fa52e5e4f4 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -24,7 +24,7 @@ from litellm.proxy.common_utils.codex_model_catalog import ( ) from litellm.types.llms.base import LiteLLMBaseModel -from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login +from .auth import cli_context_obj, context_secret_vault, get_stored_api_key, login from .claude_settings import ClaudeSettingsError, install_statusline_script from .cmd_quoting import quote_for_cmd from .pi import ( @@ -741,7 +741,7 @@ def _is_interactive() -> bool: def resolve_api_key(ctx: click.Context) -> str: - ctx_obj: Final[CliContextObj] = ctx.obj + ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"] api_key = ctx_obj.get("api_key") if api_key: @@ -765,7 +765,7 @@ _SKIP_VERIFY_HELP: Final = "Skip the pre-launch key check against the proxy." def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None: - ctx_obj: Final[CliContextObj] = ctx.obj + ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"] started_interactive: Final = _is_interactive() api_key: Final = resolve_api_key(ctx) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 3c99f402054..ad54cab6460 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -106,7 +106,7 @@ class _OptionalCliContextObjView(Protocol): def obj(self) -> CliContextObj | None: ... -def _cli_context_obj(view: _CliContextObjView) -> CliContextObj: +def cli_context_obj(view: _CliContextObjView) -> CliContextObj: return view.obj @@ -898,7 +898,7 @@ def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: """Login to LiteLLM proxy using SSO authentication""" from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER - ctx_obj: Final = _cli_context_obj(ctx) + ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"] if config_claude: settings_path: Final = claude_settings_path(os.environ) @@ -1048,7 +1048,7 @@ def print_token(ctx: click.Context): # explicitly pointed us at a server, trust whichever one `lite login` # actually issued this token for -- that's the whole point of not # needing a wrapper command. - ctx_obj: Final = _cli_context_obj(ctx) + ctx_obj: Final = cli_context_obj(ctx) issued_for_this_server: Final = token_data.get("base_url") == ctx_obj.get("base_url", "").rstrip("/") if ctx_obj.get("base_url_explicit") and not issued_for_this_server: click.echo("Not authenticated for this server. Run 'lite login'.", err=True) diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index a7fd92b9e84..fec658ac94a 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -9,6 +9,7 @@ from InquirerPy.base.control import Choice from pydantic import JsonValue, TypeAdapter, ValidationError from .... import Client +from ..auth import cli_context_obj from .config import ( DEFAULT_KEYWORD_TIER_RULES, TIER_NAMES, @@ -108,7 +109,7 @@ def _load_persisted_master_key(config_path: Path) -> str | None: def run_configure_wizard(ctx: click.Context) -> Path: """Discover the caller's accessible models, walk them through tier assignment, write config.""" - base_url: Final = ctx.obj["base_url"] + base_url: Final = cli_context_obj(ctx)["base_url"] api_key: Final = ctx.obj["api_key"] client: Final = Client(base_url=base_url, api_key=api_key) diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index f9a9356d0d6..ef73b4f0d2d 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -6,6 +6,7 @@ import click import rich from ...http_client import HTTPClient +from .auth import cli_context_obj @click.group() @@ -39,7 +40,7 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): lite encryption migrate --check # attestation scan, no writes lite encryption migrate # perform the migration """ - client: Final = HTTPClient(ctx.obj["base_url"], ctx.obj["api_key"]) + client: Final = HTTPClient(cli_context_obj(ctx)["base_url"], ctx.obj["api_key"]) if check_only: response = client.request("GET", "/credentials/migrate-encryption/check") diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index 00e8b0a3a76..d1e966d6eae 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -19,7 +19,7 @@ from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh from litellm.litellm_core_utils.private_json import ensure_private_dir from .agents import AgentRunError, resolve_api_key, verify_proxy_key -from .auth import CliContextObj, context_secret_vault, get_stored_api_key, load_token, login +from .auth import CliContextObj, cli_context_obj, context_secret_vault, get_stored_api_key, load_token, login from .claude_settings import ( BACKUP_PATH, CLAUDE_SETTINGS_PATH, @@ -125,7 +125,7 @@ def _stored_login_is_pkce(vault: SecretVault) -> bool: def ensure_fresh_login(ctx: click.Context) -> None: - ctx_obj: Final[CliContextObj] = ctx.obj + ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"].rstrip("/") vault: Final = context_secret_vault(ctx) if _usable_login(_key_resolved_on_the_way_in(ctx_obj, base_url, vault), vault): @@ -166,7 +166,7 @@ def up(ctx: click.Context) -> None: is already running (this does not start one for you). Cursor is not supported: it has no equivalent file-based config to patch. """ - ctx_obj: Final[CliContextObj] = ctx.obj + ctx_obj: Final = cli_context_obj(ctx) base_url: Final = ctx_obj["base_url"] try: diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index 08abd2febda..f6ecaf1a9be 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -9,7 +9,7 @@ import click from litellm._logging import verbose_logger -from .commands.auth import context_secret_vault +from .commands.auth import cli_context_obj, context_secret_vault ShellScope: TypeAlias = tuple[str, ...] SHELL_WORD: Final = re.compile(r"""(?P[\w-]+=)?(?P"[^"]*"|'[^']*')(?P\S*)|\S+""") @@ -120,7 +120,7 @@ def setup_shell(ctx: click.Context): show_banner() # Show server connection info - base_url: Final = ctx.obj.get("base_url") + base_url: Final = cli_context_obj(ctx).get("base_url") click.secho(f"Connected to LiteLLM server: {base_url}\n", fg="green") show_commands() diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 6af2887f14f..0f4bbc911c6 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -1,7 +1,7 @@ # stdlib imports # third party imports -from typing import Final, Protocol +from typing import Final, Literal, Protocol import click @@ -11,8 +11,8 @@ from litellm.proxy.client.health import HealthManagementClient from .commands.agents import agent_commands from .commands.auth import ( - CliContextObj, auth_group, + cli_context_obj, context_secret_vault, get_stored_api_key, keychain_unreadable_notice, @@ -39,12 +39,21 @@ from .commands.users import users from .interface import interactive_shell -class _CliContextObjView(Protocol): +class _CliContextObjSlots(Protocol): + def __setitem__( + self, + key: Literal["base_url", "api_key", "api_key_from_token_file", "base_url_explicit"], + value: str | bool | None, + /, + ) -> None: ... + + +class _WritableCliContextView(Protocol): @property - def obj(self) -> CliContextObj: ... + def obj(self) -> _CliContextObjSlots: ... -def _cli_context_obj(view: _CliContextObjView) -> CliContextObj: +def _writable_cli_context_obj(view: _WritableCliContextView) -> _CliContextObjSlots: return view.obj @@ -139,16 +148,17 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s else api_key ) - ctx.obj["base_url"] = base_url - ctx.obj["api_key"] = resolved_api_key - ctx.obj["api_key_from_token_file"] = api_key_from_token_file + ctx_obj: Final = _writable_cli_context_obj(ctx) + ctx_obj["base_url"] = base_url + ctx_obj["api_key"] = resolved_api_key + ctx_obj["api_key_from_token_file"] = api_key_from_token_file # `--base-url` defaults to localhost:4000 for local dev convenience, but # apiKeyHelper is invoked bare (no flags) -- commands that must work # unattended (print-token) need to tell "user didn't say" apart from # "user said localhost:4000 on purpose" so they can fall back to # whatever server the stored token was actually issued for. A base_url # saved via `lite config set` counts as the user saying it. - ctx.obj["base_url_explicit"] = base_url_provided or (bool(stored_base_url) and not setup_command) + ctx_obj["base_url_explicit"] = base_url_provided or (bool(stored_base_url) and not setup_command) if show_version: print_version(base_url, resolved_api_key) @@ -163,7 +173,7 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s @click.pass_context def version(ctx: click.Context): """Show the LiteLLM Proxy CLI and server version.""" - ctx_obj: Final = _cli_context_obj(ctx) + ctx_obj: Final = cli_context_obj(ctx) print_version(ctx_obj.get("base_url"), ctx_obj.get("api_key")) diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1fbf41e631b..1b50f8c24f3 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -72,7 +72,7 @@ class ModelsManagementClient: model_params: Mapping[str, object], model_info: Mapping[str, object] | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Add a new model to the proxy. @@ -115,7 +115,7 @@ class ModelsManagementClient: raise UnauthorizedError(e) raise - def delete(self, model_id: str, return_request: bool = False) -> dict[str, Any] | requests.Request: + def delete(self, model_id: str, return_request: bool = False) -> dict[str, object] | requests.Request: """ Delete a model from the proxy. @@ -249,7 +249,7 @@ class ModelsManagementClient: model_params: Mapping[str, object], model_info: Mapping[str, object] | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Update an existing model's configuration. diff --git a/litellm/proxy/client/teams.py b/litellm/proxy/client/teams.py index 54a6e869fef..dc6fa1e228a 100644 --- a/litellm/proxy/client/teams.py +++ b/litellm/proxy/client/teams.py @@ -80,7 +80,7 @@ class TeamsManagementClient: page_size: int = 10, sort_by: str | None = None, sort_order: str = "asc", - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Get a paginated list of teams with filtering and sorting options. diff --git a/litellm/proxy/client/users.py b/litellm/proxy/client/users.py index 503c92228a8..7b4f36dbc2f 100644 --- a/litellm/proxy/client/users.py +++ b/litellm/proxy/client/users.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Any, Final import requests @@ -18,7 +18,9 @@ class UsersManagementClient: headers["Authorization"] = f"Bearer {self.api_key}" return headers - def list_users(self, params: dict[str, Any] | None = None) -> list[dict[str, Any]]: + def list_users( + self, params: Mapping[str, str | bytes | int | float | Iterable[str | bytes | int | float] | None] | None = None + ) -> list[dict[str, Any]]: """List users (GET /user/list)""" url: Final = f"{self.base_url}/user/list" response: Final = requests.get(url, headers=self._get_headers(), params=params, timeout=self.timeout) @@ -27,7 +29,7 @@ class UsersManagementClient: response.raise_for_status() return response.json().get("users", response.json()) - def get_user(self, user_id: str | None = None) -> dict[str, Any]: + def get_user(self, user_id: str | None = None) -> dict[str, object]: """Get user info (GET /user/info)""" url: Final = f"{self.base_url}/user/info" params: Final = {"user_id": user_id} if user_id else {} @@ -39,7 +41,7 @@ class UsersManagementClient: response.raise_for_status() return response.json() - def get_user_v2(self, user_id: str | None = None) -> dict[str, Any]: + def get_user_v2(self, user_id: str | None = None) -> dict[str, object]: """Get user info v2 - lightweight, returns only user object (GET /v2/user/info)""" url: Final = f"{self.base_url}/v2/user/info" params: Final = {"user_id": user_id} if user_id else {} @@ -51,7 +53,7 @@ class UsersManagementClient: response.raise_for_status() return response.json() - def create_user(self, user_data: Mapping[str, object]) -> dict[str, Any]: + def create_user(self, user_data: Mapping[str, object]) -> dict[str, object]: """Create a new user (POST /user/new)""" url: Final = f"{self.base_url}/user/new" response: Final = requests.post(url, headers=self._get_headers(), json=user_data, timeout=self.timeout) @@ -60,7 +62,7 @@ class UsersManagementClient: response.raise_for_status() return response.json() - def delete_user(self, user_ids: list[str]) -> dict[str, Any]: + def delete_user(self, user_ids: list[str]) -> dict[str, object]: """Delete users (POST /user/delete)""" url: Final = f"{self.base_url}/user/delete" response: Final = requests.post( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 55cc13ff97d..5f798fce6cf 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -749,6 +749,7 @@ async def _resolve_per_request_model_group_alias( _REQUEST_MODEL: Final[TypeAdapter[str | list[str] | None]] = TypeAdapter(str | list[str] | None) +_DECODED_JSON: Final = TypeAdapter(object) _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -1834,6 +1835,10 @@ async def _team_member_within_fallback_model_budget( return True +def _content_text(fields: Mapping[str, object]) -> str: + return str(fields.get("content", "")) + + class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): self.data = data @@ -4092,7 +4097,7 @@ class ProxyBaseLLMRequestProcessing: try: d = chunk.model_dump(mode="json", exclude_none=True) if isinstance(d, dict): - str_so_far += str(d.get("content", "")) + str_so_far += _content_text(d) except Exception: pass elif isinstance(chunk, dict): @@ -4299,7 +4304,7 @@ class ProxyBaseLLMRequestProcessing: if stripped_ln.startswith("data:"): json_part = stripped_ln.split("data:", 1)[1].strip() if json_part and json_part != "[DONE]": - obj: object = json.loads(json_part) + obj: object = _DECODED_JSON.validate_python(json.loads(json_part)) if not isinstance(obj, dict): return None maybe_modified = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict( @@ -4313,24 +4318,18 @@ class ProxyBaseLLMRequestProcessing: return None @staticmethod - def _openai_stream_usage_kwargs(usage: Mapping[str, Any]) -> Mapping[str, Any]: + def _openai_stream_usage(usage: Mapping[str, Any]) -> Usage: prompt_tokens: Final = int(usage.get("prompt_tokens", 0) or 0) completion_tokens: Final = int(usage.get("completion_tokens", 0) or 0) total_tokens: Final = int( usage.get("total_tokens", prompt_tokens + completion_tokens) or (prompt_tokens + completion_tokens) ) - return MappingProxyType( - { - key: value - for key, value in ( - ("prompt_tokens", prompt_tokens), - ("completion_tokens", completion_tokens), - ("total_tokens", total_tokens), - ("completion_tokens_details", usage.get("completion_tokens_details")), - ("prompt_tokens_details", usage.get("prompt_tokens_details")), - ) - if value is not None - } + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + completion_tokens_details=usage.get("completion_tokens_details"), + prompt_tokens_details=usage.get("prompt_tokens_details"), ) @staticmethod @@ -4340,7 +4339,7 @@ class ProxyBaseLLMRequestProcessing: if obj.get("type") == "message_delta": return AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None) if obj.get("object") == "chat.completion.chunk": - return Usage(**ProxyBaseLLMRequestProcessing._openai_stream_usage_kwargs(usage)) + return ProxyBaseLLMRequestProcessing._openai_stream_usage(usage) return None @staticmethod diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 9c2cc768ae8..33633150711 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -8,7 +8,7 @@ import sys import tracemalloc from collections import Counter from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Annotated, Any, Final, NamedTuple, Protocol, TypedDict +from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query from typing_extensions import ReadOnly @@ -329,7 +329,7 @@ def _summary_process_memory(process: _ProcessHandle | None) -> tuple[_SummaryPro @router.get("/debug/memory/summary", include_in_schema=False) async def get_memory_summary( _: UserAPIKeyAuth = Depends(user_api_key_auth), -) -> dict[str, Any]: +) -> dict[str, object]: """ Get simplified memory usage summary for the proxy. @@ -700,7 +700,7 @@ async def get_memory_details( _: UserAPIKeyAuth = Depends(user_api_key_auth), top_n: int = Query(20, description="Number of top object types to return"), include_process_info: bool = Query(True, description="Include process memory info"), -) -> dict[str, Any]: +) -> dict[str, object]: """ Get detailed memory diagnostics for deep debugging. @@ -760,7 +760,7 @@ async def configure_gc_thresholds_endpoint( generation_0: int = Query(700, description="Generation 0 threshold (default: 700)"), generation_1: int = Query(10, description="Generation 1 threshold (default: 10)"), generation_2: int = Query(10, description="Generation 2 threshold (default: 10)"), -) -> dict[str, Any]: +) -> dict[str, object]: """ Configure Python garbage collection thresholds. diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 81ac9eba665..ea33bc763d4 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -6,6 +6,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status +from pydantic import TypeAdapter from starlette._utils import get_route_path from starlette.requests import HTTPConnection from typing_extensions import NotRequired, ReadOnly, Required, assert_never @@ -29,6 +30,7 @@ _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form- _BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobuf", "application/protobuf"}) _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) +_DECODED_JSON: Final = TypeAdapter(object) def resolve_inference_model( @@ -665,7 +667,7 @@ def get_tags_from_request_body(request_body: Mapping[str, object]) -> list[str]: if isinstance(metadata, str): from litellm.litellm_core_utils.safe_json_loads import safe_json_loads - parsed: Final[object] = safe_json_loads(metadata) + parsed: Final[object] = _DECODED_JSON.validate_python(safe_json_loads(metadata)) metadata = parsed if isinstance(parsed, dict) else {} elif not isinstance(metadata, dict): metadata = {} diff --git a/litellm/proxy/common_utils/proxy_state.py b/litellm/proxy/common_utils/proxy_state.py index edd18c603df..e2dd312b67d 100644 --- a/litellm/proxy/common_utils/proxy_state.py +++ b/litellm/proxy/common_utils/proxy_state.py @@ -4,7 +4,7 @@ This file is used to store the state variables of the proxy server. Example: `spend_logs_row_count` is used to store the number of rows in the `LiteLLM_SpendLogs` table. """ -from typing import Any, Literal +from typing import Literal from litellm.proxy._types import ProxyStateVariables @@ -25,12 +25,12 @@ class ProxyState: def get_proxy_state_variable( self, variable_name: valid_keys_literal, - ) -> Any: + ) -> int: return self.proxy_state_variables.get(variable_name, None) def set_proxy_state_variable( self, variable_name: valid_keys_literal, - value: Any, + value: int, ) -> None: self.proxy_state_variables[variable_name] = value diff --git a/litellm/proxy/db/base_client.py b/litellm/proxy/db/base_client.py index 127581866bb..237942016c1 100644 --- a/litellm/proxy/db/base_client.py +++ b/litellm/proxy/db/base_client.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal class CustomDB: @@ -14,12 +14,12 @@ class CustomDB: Check if key valid """ - def insert_data(self, value: Any, table_name: Literal["user", "key", "config"]): + def insert_data(self, value: object, table_name: Literal["user", "key", "config"]): """ For new key / user logic """ - def update_data(self, key: str, value: Any, table_name: Literal["user", "key", "config"]): + def update_data(self, key: str, value: object, table_name: Literal["user", "key", "config"]): """ For cost tracking logic """ diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index ca7c3341197..82c95cfe998 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -19,6 +19,8 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Final, Protocol +from typing_extensions import LiteralString, ReadOnly, TypedDict + from litellm._logging import verbose_proxy_logger from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_span import db_span @@ -91,6 +93,57 @@ _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = ( _UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60) +class _SeedTotalsRow(TypedDict): + total: ReadOnly[float | None] + before_batch: ReadOnly[float | None] + + +class _ExistingKeyRow(TypedDict): + entity_type: ReadOnly[str] + entity_id: ReadOnly[str] + window_duration: ReadOnly[str] + + +class _SeedTotalsQuery(Protocol): + async def __call__(self, query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: ... + + +class _ExistingKeysQuery(Protocol): + async def __call__(self, query: LiteralString, *args: object) -> Sequence[_ExistingKeyRow]: ... + + +class _UpsertBatch(Protocol): + def execute_raw(self, query: LiteralString, *args: object) -> None: ... + + +class _UpsertBatchManager(Protocol): + async def __aenter__(self) -> _UpsertBatch: ... + + async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + + +class _UpsertTransaction(Protocol): + def batch_(self) -> _UpsertBatchManager: ... + + +class _UpsertTransactionManager(Protocol): + async def __aenter__(self) -> _UpsertTransaction: ... + + async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ... + + +def _seed_totals_query(prisma_client: "PrismaClient") -> _SeedTotalsQuery: + return prisma_client.db.query_raw + + +def _existing_keys_query(prisma_client: "PrismaClient") -> _ExistingKeysQuery: + return prisma_client.db.query_raw + + +def _upsert_transaction(prisma_client: "PrismaClient") -> _UpsertTransactionManager: + return prisma_client.db.tx(timeout=_UPSERT_TRANSACTION_TIMEOUT) + + @dataclass(frozen=True, slots=True) class WindowSeedTotals: """The two sums a seed needs: everything persisted for the window, and the @@ -149,9 +202,9 @@ async def spend_logs_seed_totals( return None async with db_span("seed_window_spend", "LiteLLM_SpendLogs"): rows: Final = ( - await prisma_client.db.query_raw(unbounded_sql, entity_id, window_start) + await _seed_totals_query(prisma_client)(unbounded_sql, entity_id, window_start) if batch_started_at is None - else await prisma_client.db.query_raw( + else await _seed_totals_query(prisma_client)( bounded_sql, entity_id, window_start, @@ -185,7 +238,7 @@ async def _existing_primary_keys( transactions: tuple[WindowSpendTransaction, ...], ) -> frozenset[tuple[str, str, str]]: async with db_span("select_window_spend_rows", "LiteLLM_BudgetWindowSpend"): - rows: Final = await prisma_client.db.query_raw( + rows: Final = await _existing_keys_query(prisma_client)( _SELECT_EXISTING_ROWS_SQL, tuple(transaction["entity_type"] for transaction in transactions), tuple(transaction["entity_id"] for transaction in transactions), @@ -304,7 +357,7 @@ async def commit_window_spend_updates( ) async with ( db_span("commit_window_spend_updates", "LiteLLM_BudgetWindowSpend"), - prisma_client.db.tx(timeout=_UPSERT_TRANSACTION_TIMEOUT) as db_transaction, + _upsert_transaction(prisma_client) as db_transaction, db_transaction.batch_() as batcher, ): for transaction, seed_base in zip(ordered, seed_bases): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index da8a7fb0f15..166a42adfc0 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -2072,7 +2072,7 @@ class DBSpendUpdateWriter: n_retry_times, e, ) - await asyncio.sleep(random.uniform(2**attempt, 2 ** (attempt + 1))) + await asyncio.sleep(random.uniform(2.0**attempt, 2.0 ** (attempt + 1))) _handle_spend_update_failure = handle_spend_update_failure @@ -2623,7 +2623,7 @@ class DBSpendUpdateWriter: # and thus they are more likely to deadlock again. # Instead, we sleep a random amount so that they retry at slightly different times, lowering the chance of # repeated deadlocks, and therefore of exceeding the retry limit. - random.uniform(2**i, 2 ** (i + 1)) + random.uniform(2.0**i, 2.0 ** (i + 1)) ) except Exception as e: diff --git a/litellm/proxy/db/health_check_latest.py b/litellm/proxy/db/health_check_latest.py index 9deb7de4ca5..7672ed89e9a 100644 --- a/litellm/proxy/db/health_check_latest.py +++ b/litellm/proxy/db/health_check_latest.py @@ -10,11 +10,12 @@ worker on every call. ``SELECT DISTINCT ON`` keeps the transfer at one row per from __future__ import annotations import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol from pydantic import ConfigDict, JsonValue, TypeAdapter, field_validator +from typing_extensions import LiteralString from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span @@ -76,9 +77,17 @@ class LatestHealthCheckRow(LiteLLMBaseModel): _ROWS_ADAPTER: Final = TypeAdapter(tuple[LatestHealthCheckRow, ...]) +class _HealthCheckRowsQuery(Protocol): + async def __call__(self, query: LiteralString, *args: object) -> Sequence[Mapping[str, object]]: ... + + +def _health_check_rows_query(prisma_client: PrismaClient) -> _HealthCheckRowsQuery: + return prisma_client.db.query_raw + + async def query_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]: async with db_span("latest_health_checks", "LiteLLM_HealthCheckTable"): - rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL) + rows: Final = await _health_check_rows_query(prisma_client)(LATEST_HEALTH_CHECKS_SQL) return _ROWS_ADAPTER.validate_python(rows) @@ -97,7 +106,9 @@ async def fetch_latest_health_checks_for_models( return () try: async with db_span("latest_health_checks", "LiteLLM_HealthCheckTable"): - rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, list(model_names)) + rows: Final = await _health_check_rows_query(prisma_client)( + LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, list(model_names) + ) return _ROWS_ADAPTER.validate_python(rows) except Exception as query_err: # noqa: BLE001 # a paged model list must not fail on its health decoration verbose_proxy_logger.error("Error getting latest health checks for models: %s", query_err) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 0cbce6339fd..78da03150dc 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -13,7 +13,9 @@ import urllib import urllib.parse from collections.abc import Awaitable, Callable, Coroutine from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar +from typing import TYPE_CHECKING, Final, Protocol, TypeVar + +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span @@ -33,6 +35,7 @@ from litellm.secret_managers.main import str_to_bool if TYPE_CHECKING: from prisma import Prisma + from prisma.types import HttpConfig __all__ = ( "IAMEndpoint", @@ -42,6 +45,9 @@ __all__ = ( ) +_ENGINE_ATTRIBUTE: Final = TypeAdapter(object) + + class _PrismaProcess(Protocol): pid: object @@ -116,7 +122,7 @@ class _TrackedPrismaEngine: return self._engine.process def __getattr__(self, name: str) -> object: - return getattr(self._engine, name) + return _ENGINE_ATTRIBUTE.validate_python(getattr(self._engine, name)) async def query(self, content: str, *, tx_id: str | None) -> object: return await self._finish_even_if_cancelled(self._tracked_query(content, tx_id)) @@ -240,7 +246,7 @@ class PrismaWrapper: def __init__( self, - original_prisma: Any, + original_prisma: "Prisma", iam_token_db_auth: bool = False, *, token_auth: DatabaseTokenAuth | None = None, @@ -590,7 +596,7 @@ class PrismaWrapper: async def recreate_prisma_client( self, new_db_url: str, - http_client: object | None = None, + http_client: "HttpConfig | None" = None, *, expected_generation: int | None = None, ) -> bool: @@ -628,7 +634,7 @@ class PrismaWrapper: async def _recreate_prisma_client_locked( self, new_db_url: str, - http_client: object | None = None, + http_client: "HttpConfig | None" = None, *, expected_generation: int | None = None, ) -> bool: @@ -639,6 +645,7 @@ class PrismaWrapper: lock) don't re-acquire it — `asyncio.Lock` is not reentrant. """ from prisma import Prisma + from prisma.types import DatasourceOverride if expected_generation is not None and expected_generation != self._engine_generation: verbose_proxy_logger.info( @@ -665,12 +672,15 @@ class PrismaWrapper: self._expected_engine_deaths.add(old_engine_pid) await self._kill_engine_process(old_engine_pid) - kwargs: Final[dict[str, Any]] = {} - if http_client is not None: - kwargs["http"] = http_client - if self._recreate_uses_datasource: - kwargs["datasource"] = {"url": new_db_url} - self._original_prisma = Prisma(**kwargs) + datasource: Final = DatasourceOverride(url=new_db_url) if self._recreate_uses_datasource else None + if http_client is not None and datasource is not None: + self._original_prisma = Prisma(http=http_client, datasource=datasource) + elif http_client is not None: + self._original_prisma = Prisma(http=http_client) + elif datasource is not None: + self._original_prisma = Prisma(datasource=datasource) + else: + self._original_prisma = Prisma() await self._original_prisma.connect() self._active_drain_tracker = self._instrument_prisma_client(self._original_prisma) @@ -885,6 +895,12 @@ class PrismaWrapper: seconds_left: Final = (expiration_time - datetime.utcnow()).total_seconds() return seconds_left > self.TOKEN_REFRESH_BUFFER_SECONDS + if TYPE_CHECKING: + + def is_connected(self) -> bool: ... + + async def disconnect(self, timeout: float | timedelta | None = None) -> None: ... + def __getattr__(self, name: str): """ Proxy attribute access to the underlying Prisma client. diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py index 02015c699c9..673b549b645 100644 --- a/litellm/proxy/db/proxy_worker_heartbeat.py +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -13,10 +13,10 @@ still agree. from __future__ import annotations import socket -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import LiteralString, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -58,6 +58,14 @@ class _LiveWorkerCountRow(TypedDict): _COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...]) +class _HeartbeatExecute(Protocol): + async def __call__(self, query: LiteralString, *args: object) -> int: ... + + +def _heartbeat_execute(prisma_client: PrismaClient) -> _HeartbeatExecute: + return prisma_client.db.execute_raw + + class ProxyWorkerHeartbeat: def __init__(self, prisma_client: PrismaClient, worker_id: str | None = None) -> None: self.prisma_client: Final = prisma_client @@ -67,16 +75,16 @@ class ProxyWorkerHeartbeat: async def beat(self) -> None: try: async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"): - await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname) + _ = await _heartbeat_execute(self.prisma_client)(BEAT_SQL, self.worker_id, self.hostname) async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"): - await self.prisma_client.db.execute_raw(PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) + _ = await _heartbeat_execute(self.prisma_client)(PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err) async def deregister(self) -> None: try: async with db_span("deregister_proxy_worker", "LiteLLM_ProxyWorkerHeartbeat"): - await self.prisma_client.db.execute_raw(DEREGISTER_SQL, self.worker_id) + _ = await _heartbeat_execute(self.prisma_client)(DEREGISTER_SQL, self.worker_id) except Exception as deregister_err: # noqa: BLE001 # best-effort cleanup; the liveness window ages the row out anyway verbose_proxy_logger.debug("Proxy worker heartbeat deregister failed: %s", deregister_err) diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index 0eb378b2fe9..48ad57fd06e 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -9,6 +9,8 @@ from collections.abc import Callable from datetime import timedelta from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.proxy.db.prisma_client import PrismaWrapper @@ -34,6 +36,8 @@ _MODEL_READ_METHODS: Final = frozenset( # Top-level Prisma client methods that read from the database. _TOP_LEVEL_READ_METHODS: Final = frozenset({"query_first", "query_raw"}) +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) + class _RoutedActions: """Per-model accessor that sends reads to the reader and writes to the writer. @@ -57,8 +61,8 @@ class _RoutedActions: def __getattr__(self, name: str) -> object: if name in _MODEL_READ_METHODS and self._should_use_reader(): - return getattr(self._reader_actions, name) - return getattr(self._writer_actions, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._reader_actions, name)) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._writer_actions, name)) class WriterPinnedClient: @@ -295,13 +299,13 @@ class RoutingPrismaWrapper: def __getattr__(self, name: str) -> Any: if name in _TOP_LEVEL_READ_METHODS: return getattr(self.read_target, name) - writer_attr: Final[object] = getattr(self._writer, name) + writer_attr: Final = _DELEGATED_ATTRIBUTE.validate_python(getattr(self._writer, name)) # Per-model action accessors are non-callable instances that expose # both `find_many` and `create`. Methods like execute_raw / batch_ / # tx are callables and stay on the writer untouched. if not callable(writer_attr) and hasattr(writer_attr, "find_many") and hasattr(writer_attr, "create"): try: - reader_attr: Final[object] = getattr(self._reader, name) + reader_attr: Final = _DELEGATED_ATTRIBUTE.validate_python(getattr(self._reader, name)) except AttributeError: return writer_attr return _RoutedActions(writer_attr, reader_attr, self._should_use_reader) diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index cef90eb89c2..2e3530d7613 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -330,8 +330,8 @@ async def list_overrides_for_tool( ) for perm in perms: op_id = getattr(perm, "object_permission_id", None) or "" - tokens = getattr(perm, "verification_tokens", []) or [] - teams = getattr(perm, "teams", []) or [] + tokens: Sequence[object] = getattr(perm, "verification_tokens", []) or [] + teams: Sequence[object] = getattr(perm, "teams", []) or [] for t in tokens: out.append( ToolPolicyOverrideRow( diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index edb7e043ec5..d713a18f470 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, Union, from urllib.parse import urlparse from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -89,6 +89,7 @@ if TYPE_CHECKING: router: Final = APIRouter() GUARDRAIL_REGISTRY: Final = GuardrailRegistry() +_DECODED_JSON: Final = TypeAdapter(object) def _as_str_object_mapping(mapping: Mapping[str, object]) -> Mapping[str, object]: @@ -1619,7 +1620,7 @@ async def get_major_airlines(): with open(airlines_path, "r", encoding="utf-8") as f: import json - airlines: Final = json.load(f) + airlines: Final = _DECODED_JSON.validate_python(json.load(f)) return {"airlines": airlines} except Exception as e: raise HTTPException(status_code=500, detail=f"Error reading major_airlines.json: {e}") from e diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 3967ef7152c..885ab59f5f7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -6,7 +6,7 @@ Azure Prompt Shield Native Guardrail Integrationfor LiteLLM import math from collections.abc import Mapping, MutableMapping from contextvars import ContextVar -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NoReturn, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NoReturn, cast from fastapi import HTTPException @@ -35,6 +35,7 @@ from .base import ( # noqa: F401 # legacy module exports ) if TYPE_CHECKING: + from litellm.caching.caching import DualCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import LitellmParams @@ -240,10 +241,10 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai async def async_pre_call_hook( self, user_api_key_dict: "UserAPIKeyAuth", - cache: Any, - data: dict[str, Any], + cache: "DualCache", + data: dict[str, object], call_type: CallTypesLiteral, - ) -> dict[str, Any] | None: + ) -> dict[str, object] | None: """ Pre-call hook to scan user prompts before sending to LLM. diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 86ce8cb5c09..b92020c89be 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -20,7 +20,7 @@ from collections.abc import AsyncGenerator, Mapping, Sequence from datetime import datetime, timezone from itertools import accumulate, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, Optional, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, Optional, Protocol, cast import httpx from fastapi import HTTPException @@ -117,6 +117,7 @@ _BEDROCK_TOO_LARGE_ERROR_SUBSTRINGS: Final = ( _BEDROCK_APPLY_GUARDRAIL_MAX_THROTTLE_RETRIES: Final = 3 _BEDROCK_APPLY_GUARDRAIL_BASE_BACKOFF_SECONDS: Final = 0.5 _BEDROCK_WHITESPACE: Final = re.compile(r"\s") +_DECODED_JSON: Final = TypeAdapter(object) _NO_TRACING_DETAIL: Final[GuardrailTracingDetail] = {} # Resource-less, detect-only InvokeGuardrailChecks API (no guardrail resource required). _BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke" @@ -257,6 +258,14 @@ def _is_responses_api_route(request_route: str | None) -> bool: return call_types is not None and any(call_type in _RESPONSES_API_CALL_TYPES for call_type in call_types) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # During-call must use async_moderation_hook (not unified apply_guardrail), otherwise # OpenAI translation always passes input_type="request" and spend/UI show PRE-CALL. @@ -2158,11 +2167,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): try: raw: Final = getattr(response, "content", None) if isinstance(raw, (bytes, bytearray)): - payload = json.loads(raw.decode("utf-8")) + payload = _DECODED_JSON.validate_python(json.loads(raw.decode("utf-8"))) else: text: Final = getattr(response, "text", None) if isinstance(text, str): - payload = json.loads(text) + payload = _DECODED_JSON.validate_python(json.loads(text)) except Exception: # Can't parse -> assume no explicit Exception marker return False @@ -2207,7 +2216,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): status_code: Final = response.status_code message: Final = "Bedrock guardrail request failed" try: - body: Final = response.json() + body: Final = _json_body(response) except Exception: text: Final = getattr(response, "text", None) or "" if isinstance(text, str) and text.strip(): diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index bf3ca71f45c..029feba89fb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -22,7 +22,7 @@ import json import time from collections import Counter, OrderedDict from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeGuard from urllib.parse import urlparse import httpx @@ -144,6 +144,14 @@ def _validate_api_base(url: str) -> str: return url +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip return isinstance(value, dict) @@ -794,7 +802,7 @@ class CompresrGuardrail(CustomGuardrail): return None try: - body: Final[object] = raw_response.json() + body: Final = _json_body(raw_response) except (ValueError, httpx.DecodingError, RecursionError): # RecursionError: a deeply nested JSON body overflows the parser; # route it through the fail policy rather than let it escape as a 500. diff --git a/litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py b/litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py index 50a14375107..d507df24faa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py @@ -6,7 +6,7 @@ # +-------------------------------------------------------------+ import os -from collections.abc import AsyncGenerator, AsyncIterable +from collections.abc import AsyncGenerator, AsyncIterable, Mapping from datetime import datetime from typing import Final @@ -189,7 +189,7 @@ class DynamoAIGuardrails(CustomGuardrail): applied_policies: Final = response.get("appliedPolicies", []) violations_detected: Final[list[str]] = [] - violation_details: Final[dict[str, object]] = {} + violation_details: Final[dict[str, Mapping[str, object]]] = {} # For now, only handle BLOCK action if final_action == "BLOCK": diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index aa10a80f815..dac802a1ddc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -7,7 +7,7 @@ import time import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard import httpx from fastapi import HTTPException @@ -71,6 +71,14 @@ _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 _REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip return isinstance(value, dict) @@ -634,7 +642,7 @@ class HeadroomGuardrail(CustomGuardrail): ) try: - body: Final[object] = response.json() + body: Final = _json_body(response) except ValueError: return _CompressResult( self._handle_compress_failure( @@ -754,7 +762,7 @@ class HeadroomGuardrail(CustomGuardrail): return f"[Headroom: retrieval error {raw_response.status_code} for hash={hash_value}]" try: - body: Final[object] = raw_response.json() + body: Final = _json_body(raw_response) except ValueError: return raw_response.text diff --git a/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py b/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py index 380562aacac..83dd90dafb4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py +++ b/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py @@ -6,9 +6,9 @@ # +-------------------------------------------------------------+ import os -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, AsyncIterable, Mapping from datetime import datetime -from typing import Any, Final +from typing import Final import httpx @@ -647,7 +647,7 @@ class IBMGuardrailDetector(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: Any, + response: AsyncIterable[ModelResponseStream], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 3188febf59a..f959292d567 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast import yaml from fastapi import HTTPException +from pydantic import TypeAdapter from litellm import Router from litellm._logging import verbose_proxy_logger @@ -74,6 +75,7 @@ from .competitor_intent import ( from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 +_DECODED_ARGUMENTS: Final = TypeAdapter(object) GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b") SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+") @@ -1851,7 +1853,7 @@ class ContentFilterGuardrail(CustomGuardrail): detections: list[ContentFilterDetection], # mutable-ok: _filter_single_text appends into a caller-owned list ) -> str: try: - parsed: Final[object] = json.loads(arguments) + parsed: Final = _DECODED_ARGUMENTS.validate_python(json.loads(arguments)) except (json.JSONDecodeError, TypeError, ValueError): return self._filter_single_text(arguments, detections=detections) if not isinstance(parsed, (dict, list)): diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index c970ac263e7..06ea3ce4b20 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -5,7 +5,7 @@ import uuid from collections.abc import AsyncGenerator, AsyncIterable, Iterator, Mapping, Sequence from typing import ( TYPE_CHECKING, - Any, # noqa: TID251 # **kwargs forwards verbatim to CustomGuardrail.__init__ + Any, # noqa: TID251 # the streaming hook and the flush copy pass provider chunks through untyped ClassVar, Final, Literal, @@ -15,6 +15,7 @@ from typing import ( import httpx from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -22,6 +23,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import attribute_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -93,6 +95,10 @@ def _string_tool_arguments(calls: Sequence[object]) -> Iterator[tuple[object, st yield function, arguments +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _ContentDelta(Protocol): content: object @@ -123,7 +129,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): guardrail_name: str = GUARDRAIL_NAME, api_base: str | None = None, api_key: str | None = None, - **kwargs: Any, # noqa: LIT008 # kwargs-ok: forwarded verbatim to CustomGuardrail.__init__ + **kwargs: Unpack[_CustomGuardrailOptions], # noqa: LIT008 # kwargs-ok: forwarded verbatim to CustomGuardrail.__init__ ) -> None: self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) env_base: Final = os.environ.get("LLM_SHIELD_PROXY_API_BASE") @@ -366,13 +372,11 @@ class LLMShieldProxyGuardrail(CustomGuardrail): if response_slots: return await self._restore_responses_api_response(response, response_slots, data) - choices: Final = getattr(response, "choices", None) + choices: Final = attribute_items(response, "choices") if not choices: return response pending: Final[SlotSink] = [] - choice: object - tool_call: object for choice in choices: message: object = getattr(choice, "message", None) if message is None: @@ -383,7 +387,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): content = getattr(message, "content", None) if isinstance(content, str) and content: pending.append((content, functools.partial(setattr, message, "content"))) - for tool_call in getattr(message, "tool_calls", None) or (): + for tool_call in attribute_items(message, "tool_calls"): function: object = getattr(tool_call, "function", None) arguments = getattr(function, "arguments", None) if function is not None else None if isinstance(arguments, str) and arguments: @@ -450,8 +454,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): function_call_output holds `output` -- so the two directions stay symmetric. """ slots: Final[SlotSink] = [] - item: object - for item in getattr(response, "output", None) or (): + for item in attribute_items(response, "output"): collect_response_item(item, slots) return tuple(slots) @@ -504,8 +507,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): continue restored_chunk = detached(chunk) last_chunk = restored_chunk - choice: object - for choice in getattr(restored_chunk, "choices", None) or (): + for choice in attribute_items(restored_chunk, "choices"): await self._restore_choice(choice, carries, session_id) yield restored_chunk @@ -536,8 +538,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): await self._restore_content_window(delta, (index, None), carries, session_id, is_final) - tool_call: object - for tool_call in getattr(delta, "tool_calls", None) or (): + for tool_call in attribute_items(delta, "tool_calls"): await self._restore_tool_call_window(tool_call, index, carries, session_id) if is_final: diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py index fbc83f00dba..72474ba27c3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/purview_dlp.py @@ -48,6 +48,13 @@ if TYPE_CHECKING: ) +def _header_value(headers: httpx.Headers, name: str) -> str | None: + try: + return headers[name] + except KeyError: + return None + + class MicrosoftPurviewDLPGuardrail(PurviewGuardrailBase, CustomGuardrail): """ Microsoft Purview DLP guardrail. @@ -150,7 +157,7 @@ class MicrosoftPurviewDLPGuardrail(PurviewGuardrailBase, CustomGuardrail): upstream_status: Final = exc.response.status_code client_status: Final = 502 if upstream_status in (401, 403) else upstream_status headers: dict[str, str] | None = None - retry_after: Final[str | None] = exc.response.headers.get("retry-after") + retry_after: Final = _header_value(exc.response.headers, "retry-after") if retry_after: headers = {"Retry-After": retry_after} raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 21c94b76d21..e590e5c2e43 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -1,11 +1,11 @@ import time -from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence +from collections.abc import AsyncGenerator, Awaitable, Callable, Generator, Mapping, Sequence from enum import Enum, auto -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Protocol, runtime_checkable import httpx from fastapi import HTTPException -from pydantic import ConfigDict, RootModel, with_config +from pydantic import ConfigDict, RootModel, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict if TYPE_CHECKING: @@ -66,6 +66,12 @@ from litellm.types.utils import ( ) GUARDRAIL_NAME: Final = "model_armor" +_RESPONSE_JSON: Final = TypeAdapter(object) + + +@runtime_checkable +class _AwaitableJSON(Protocol): + def __await__(self) -> Generator[object, None, object]: ... @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) @@ -411,8 +417,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): ) raise ModelArmorAPIError(detail) - json_response: Final = response.json() - if hasattr(json_response, "__await__"): + json_response: Final = _RESPONSE_JSON.validate_python(response.json()) + if isinstance(json_response, _AwaitableJSON): return _ModelArmorResponseRoot.model_validate(await json_response).root return _ModelArmorResponseRoot.model_validate(json_response).root diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 0607996d13d..c239ac9e938 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -126,6 +126,7 @@ class _ResponsesInputItem(LiteLLMBaseModel): _ResponsesInput: TypeAlias = str | tuple[_ResponsesInputItem, ...] | None _RESPONSES_INPUT: Final[TypeAdapter[_ResponsesInput]] = TypeAdapter(_ResponsesInput) +_DECODED_ARGUMENTS: Final = TypeAdapter(object) if TYPE_CHECKING: @@ -640,7 +641,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): # If the original args were structured, preserve the type. if isinstance(original_args, (dict, list)): try: - parsed: Final[object] = json.loads(masked_text) + parsed: Final = _DECODED_ARGUMENTS.validate_python(json.loads(masked_text)) except (json.JSONDecodeError, TypeError): raise HTTPException( status_code=400, diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 0c1be621923..01a9259203a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -30,6 +30,7 @@ from litellm.constants import ( PRESIDIO_ANALYZE_CHUNK_CONCURRENCY, PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS, ) +from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -93,6 +94,10 @@ async def _json_body(response: _JsonResponse) -> object: return await response.json() +def _write_attribute(target: object, name: str, value: object) -> None: + setattr(target, name, value) + + _LoopSemaphores = dict[asyncio.AbstractEventLoop, asyncio.Semaphore] @@ -1373,38 +1378,41 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) # 2. Process tool calls - tool_calls = getattr(message, "tool_calls", None) - if tool_calls: - tool_call: object - for tool_call in tool_calls: - function = getattr(tool_call, "function", None) - if function and hasattr(function, "arguments"): - args = function.arguments - if isinstance(args, str): - if mode == "unmask": - function.arguments = self._unmask_pii_text(args, pii_tokens) - elif mode == "mask": - function.arguments = await self.check_pii( - text=args, - output_parse_pii=False, - presidio_config=presidio_config, - request_data=request_data, - ) - - # 3. Process legacy function calls - function_call = getattr(message, "function_call", None) - if function_call and hasattr(function_call, "arguments"): - args = function_call.arguments + for tool_call in attribute_items(message, "tool_calls"): + function = attribute_or_none(tool_call, "function") + args = attribute_or_none(function, "arguments") if function else None if isinstance(args, str): if mode == "unmask": - function_call.arguments = self._unmask_pii_text(args, pii_tokens) + _write_attribute(function, "arguments", self._unmask_pii_text(args, pii_tokens)) elif mode == "mask": - function_call.arguments = await self.check_pii( - text=args, + _write_attribute( + function, + "arguments", + await self.check_pii( + text=args, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ), + ) + + # 3. Process legacy function calls + function_call = attribute_or_none(message, "function_call") + legacy_args = attribute_or_none(function_call, "arguments") if function_call else None + if isinstance(legacy_args, str): + if mode == "unmask": + _write_attribute(function_call, "arguments", self._unmask_pii_text(legacy_args, pii_tokens)) + elif mode == "mask": + _write_attribute( + function_call, + "arguments", + await self.check_pii( + text=legacy_args, output_parse_pii=False, presidio_config=presidio_config, request_data=request_data, - ) + ), + ) return response async def _mask_output_response( diff --git a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py index f8e6f98f893..f2f3ec201f5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py +++ b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py @@ -7,7 +7,7 @@ import random from collections.abc import Iterable, Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol from urllib.parse import urlsplit import httpx @@ -27,6 +27,7 @@ from litellm.integrations.custom_guardrail import ( get_session_id_from_request_data, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages from litellm.llms.custom_httpx.http_handler import ( @@ -176,6 +177,14 @@ def _error_response_text(response: httpx.Response) -> str: return "" +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + def _as_dict(value: object) -> dict: return value if isinstance(value, dict) else {} @@ -318,7 +327,7 @@ def _choice_terminal_reason(choice: object) -> str | None: ) -def _response_finish_reason(response: Any) -> str | None: +def _response_finish_reason(response: object) -> str | None: if response is None: return None if isinstance(response, dict): @@ -1083,7 +1092,7 @@ class StraikerGuardrail(CustomGuardrail): if last_failure is None or not last_failure.retryable: return parsed, last_failure if attempt < attempts - 1: - backoff: float = min(self.initial_backoff * (2**attempt), self.max_backoff) + backoff: float = min(self.initial_backoff * (1 << attempt), self.max_backoff) await asyncio.sleep(random.uniform(0, backoff)) return None, last_failure or _WebhookFailure("unknown error", is_unreachable=True) @@ -1107,13 +1116,16 @@ class StraikerGuardrail(CustomGuardrail): def _parse_verdict(self, resp: httpx.Response) -> tuple[StraikerWebhookResponse | None, _WebhookFailure | None]: try: - body: Final = resp.json() - if not isinstance(body, Mapping): + body: Final = _json_body(resp) + body_mapping: Final = as_str_mapping(body) + if body_mapping is None: return None, _WebhookFailure( f"invalid response schema: expected an object, got {type(body).__name__}", is_unreachable=False ) parsed: Final = ( - _v3_response(body) if self.api_version == "v3" else StraikerWebhookResponse.model_validate(body) + _v3_response(body_mapping) + if self.api_version == "v3" + else StraikerWebhookResponse.model_validate(body_mapping) ) except (ValidationError, json.JSONDecodeError) as ve: return None, _WebhookFailure(f"invalid response schema: {ve}", is_unreachable=False) diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 94e48762101..ebd50c8cd39 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -45,6 +45,7 @@ from litellm.types.utils import ( ) GUARDRAIL_NAME: Final = "tool_permission" +_DECODED_ARGUMENTS: Final = TypeAdapter(object) _REQUEST_DATA: Final = TypeAdapter(object) @@ -326,7 +327,7 @@ class ToolPermissionGuardrail(CustomGuardrail): parsed_arguments: object = {} try: if isinstance(arguments, str): - parsed_arguments = json.loads(arguments) + parsed_arguments = _DECODED_ARGUMENTS.validate_python(json.loads(arguments)) elif isinstance(arguments, dict): parsed_arguments = arguments else: diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py index 59001c8bc8d..de0f8856522 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py @@ -47,6 +47,10 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _attribute(source: object, name: str) -> object: + return getattr(source, name, None) + + def _get_request_object_permission_ids( request_data: dict, ) -> tuple[str | None, str | None]: @@ -59,8 +63,8 @@ def _get_request_object_permission_ids( continue auth = meta.get("user_api_key_auth") if auth is not None and hasattr(auth, "object_permission_id"): - key_op = getattr(auth, "object_permission_id", None) - team_op = getattr(auth, "team_object_permission_id", None) + key_op = _attribute(auth, "object_permission_id") + team_op = _attribute(auth, "team_object_permission_id") if key_op is not None or team_op is not None: return ( str(key_op).strip() if key_op else None, diff --git a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py index b7d83b407a4..da81f3088b2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py @@ -24,8 +24,9 @@ Design notes (intentional divergences from the framework defaults): import asyncio import os +from collections.abc import Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, Optional +from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi.exceptions import HTTPException from pydantic import TypeAdapter @@ -36,7 +37,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys +from litellm.litellm_core_utils.core_helpers import attribute_or_none, iter_items, redact_nested_match_and_regex_keys from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -58,6 +59,10 @@ if TYPE_CHECKING: ) +def _dict_fields(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, dict) else None + + def _sanitize_scan_result_for_logging(scan_result: dict) -> dict: without_secrets: Final = {key: value for key, value in scan_result.items() if key != "secret_fields"} redacted: Final = redact_nested_match_and_regex_keys(without_secrets) @@ -466,18 +471,19 @@ class XecGuardGuardrail(CustomGuardrail): return None @staticmethod - def _extract_assistant_text_from_response(response: Any) -> str | None: + def _extract_assistant_text_from_response(response: object) -> str | None: if response is None: return None - choices = None + response_fields: Final = _dict_fields(response) + choices: object = None if hasattr(response, "choices"): - choices = response.choices - elif isinstance(response, dict): - choices = response.get("choices") + choices = attribute_or_none(response, "choices") + elif response_fields is not None: + choices = response_fields.get("choices") if not choices: return None text_parts: Final[list[str]] = [] - for choice in choices: + for choice in iter_items(choices): content = XecGuardGuardrail._extract_choice_content(choice) text = XecGuardGuardrail._content_to_text(content) if text: @@ -485,19 +491,22 @@ class XecGuardGuardrail(CustomGuardrail): return "\n".join(text_parts) or None @staticmethod - def _extract_choice_content(choice: Any) -> object: + def _extract_choice_content(choice: object) -> object: + choice_fields: Final = _dict_fields(choice) + message: object if hasattr(choice, "message"): - message = choice.message - elif isinstance(choice, dict): - message = choice.get("message") + message = attribute_or_none(choice, "message") + elif choice_fields is not None: + message = choice_fields.get("message") else: return None if message is None: return None + message_fields: Final = _dict_fields(message) if hasattr(message, "content"): - return message.content - if isinstance(message, dict): - return message.get("content") + return attribute_or_none(message, "content") + if message_fields is not None: + return message_fields.get("content") return None @staticmethod diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 7e11b69108b..1ddbc139564 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -10,10 +10,10 @@ from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Se from datetime import datetime, timezone from functools import partial from itertools import groupby -from operator import itemgetter from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -34,6 +34,7 @@ if TYPE_CHECKING: _UPSERT_RETRY_TIMES: Final = 3 _MAX_PENDING_ROWS: Final = 10_000 +_DECODED_METADATA: Final = TypeAdapter(object) _RowKey = TypeVar("_RowKey") _RowValue = TypeVar("_RowValue") @@ -55,6 +56,10 @@ class _UsageUnitIncrement(NamedTuple): """Units recorded with no known price, the share cost leaves out.""" +def _usage_unit_key(entry: tuple[_UsageUnitKey, _UsageUnitIncrement]) -> _UsageUnitKey: + return entry[0] + + def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement: if cost is None: return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units) @@ -188,7 +193,7 @@ async def _upsert_rows_with_retry( outcomes[key], ) return retryable - await sleep(2 ** (_UPSERT_RETRY_TIMES - retries_left)) + await sleep(2.0 ** (_UPSERT_RETRY_TIMES - retries_left)) return await _upsert_rows_with_retry(retryable, upsert_row, label, sleep, retries_left - 1) @@ -213,7 +218,7 @@ def _parse_guardrail_info_from_payload(payload: Mapping[str, object]) -> Sequenc return [] if isinstance(meta, str): try: - meta = json.loads(meta) + meta = _DECODED_METADATA.validate_python(json.loads(meta)) except (json.JSONDecodeError, TypeError): return [] if not isinstance(meta, dict): @@ -269,11 +274,11 @@ def _iter_usage_unit_increments( def _sum_usage_unit_increments( logs_to_process: Sequence[Mapping[str, object]], ) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]: - ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=itemgetter(0)) + ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=_usage_unit_key) return MappingProxyType( { key: _summed_increments(increment for _, increment in group) - for key, group in groupby(ordered, key=itemgetter(0)) + for key, group in groupby(ordered, key=_usage_unit_key) } ) diff --git a/litellm/proxy/hooks/batch_enqueued_tokens.py b/litellm/proxy/hooks/batch_enqueued_tokens.py index 2a1719ed4c4..da595e101f1 100644 --- a/litellm/proxy/hooks/batch_enqueued_tokens.py +++ b/litellm/proxy/hooks/batch_enqueued_tokens.py @@ -107,6 +107,7 @@ _RESERVE_RESULT_ADAPTER: Final = TypeAdapter(tuple[int, int]) _POPPED_VALUE_ADAPTER: Final = TypeAdapter(str | bytes | None) _STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None) _RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation) +_LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object) class _ScriptRunner(Protocol): @@ -445,10 +446,12 @@ class BatchEnqueuedTokenStore: async def _pop_local_record(self, batch_id: str, span: "Span | None") -> object: async with self._lock: - stored = await self.internal_usage_cache.async_get_cache( - key=self._record_key(batch_id), - litellm_parent_otel_span=span, - local_only=True, + stored: Final = _LOCAL_RECORD_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=self._record_key(batch_id), + litellm_parent_otel_span=span, + local_only=True, + ) ) if stored is None: return None @@ -456,12 +459,16 @@ class BatchEnqueuedTokenStore: return stored async def _get_local_counter(self, scope: BatchEnqueuedTokenScope, span: "Span | None") -> int: - stored = await self.internal_usage_cache.async_get_cache( - key=self._counter_key(scope), - litellm_parent_otel_span=span, - local_only=True, + return ( + _STORED_COUNTER_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=self._counter_key(scope), + litellm_parent_otel_span=span, + local_only=True, + ) + ) + or 0 ) - return _STORED_COUNTER_ADAPTER.validate_python(stored) or 0 async def _set_local_counter(self, scope: BatchEnqueuedTokenScope, value: int, span: "Span | None") -> None: await self.internal_usage_cache.async_set_cache( diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index d3ca4597f0d..19251b53f13 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -833,10 +833,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): key_remaining_tpm_limit = user_api_key_dict.tpm_limit - current["current_tpm"] key_tpm_limit = user_api_key_dict.tpm_limit - if hasattr(response, "_hidden_params"): - _hidden_params = getattr(response, "_hidden_params") - else: - _hidden_params = None + _hidden_params = getattr(response, "_hidden_params", None) if _hidden_params is not None and (isinstance(_hidden_params, BaseModel) or isinstance(_hidden_params, dict)): if isinstance(_hidden_params, BaseModel): _hidden_params = _hidden_params.model_dump() diff --git a/litellm/proxy/hooks/sensitive_data_routing.py b/litellm/proxy/hooks/sensitive_data_routing.py index 0909f36dbe5..6e2bd393ef6 100644 --- a/litellm/proxy/hooks/sensitive_data_routing.py +++ b/litellm/proxy/hooks/sensitive_data_routing.py @@ -14,6 +14,8 @@ import logging import os from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -32,6 +34,7 @@ else: SENSITIVE_ROUTING_CACHE_PREFIX: Final = "sensitive_route" DEFAULT_SENSITIVE_ROUTING_TTL: Final = 3600 +_CACHED_ROUTE: Final = TypeAdapter(object) class _PROXY_SensitiveDataRoutingHandler(CustomLogger): @@ -87,9 +90,11 @@ class _PROXY_SensitiveDataRoutingHandler(CustomLogger): if self.internal_usage_cache.dual_cache.redis_cache is not None: try: - result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) - if result is not None: - routed_model: Final = str(result) + redis_route: Final = _CACHED_ROUTE.validate_python( + await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) + ) + if redis_route is not None: + routed_model: Final = str(redis_route) remaining_ttl = await self.internal_usage_cache.dual_cache.redis_cache.async_get_ttl(key=cache_key) await self.internal_usage_cache.async_set_cache( key=cache_key, @@ -107,13 +112,15 @@ class _PROXY_SensitiveDataRoutingHandler(CustomLogger): e, ) - result = await self.internal_usage_cache.async_get_cache( - key=cache_key, - litellm_parent_otel_span=None, - local_only=True, + local_route: Final = _CACHED_ROUTE.validate_python( + await self.internal_usage_cache.async_get_cache( + key=cache_key, + litellm_parent_otel_span=None, + local_only=True, + ) ) - if result is not None: - return str(result) + if local_route is not None: + return str(local_route) return None @with_service_target("sensitive_route_pins") diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ae381775391..6bd31a42e9e 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -87,6 +87,7 @@ from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJE _SPECIAL_HEADERS_CACHE: Final = frozenset(str(v.value).lower() for v in SpecialHeaders) _REDACTED_HEADER_VALUE: Final = "***REDACTED***" +_DECODED_JSON: Final = TypeAdapter(object) _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names() | frozenset( {"cookie", "proxy-authorization"} ) @@ -503,7 +504,7 @@ def _normalized_metadata_object(field: str, value: object) -> Mapping[str, objec """ if isinstance(value, dict): return value - if isinstance(value, str) and isinstance((parsed := safe_json_loads(value)), dict): + if isinstance(value, str) and isinstance((parsed := _DECODED_JSON.validate_python(safe_json_loads(value))), dict): return parsed raise _invalid_metadata_type_error(field=field, value=value) @@ -585,13 +586,17 @@ def _key_or_team_allows_client_pricing_override( ) +def _opts_out_of_message_redaction(fields: Mapping[str, object]) -> bool: + return "turn_off_message_logging" in fields and _is_false_like(fields["turn_off_message_logging"]) + + def _strip_client_message_redaction_opt_out(data: dict[str, object]) -> None: stripped: Final[list[str]] = [] - if "turn_off_message_logging" in data and _is_false_like(data["turn_off_message_logging"]): + if _opts_out_of_message_redaction(data): stripped.append("turn_off_message_logging") data.pop("turn_off_message_logging", None) for slot_label, metadata in iter_client_callback_metadata_dicts(data): - if "turn_off_message_logging" in metadata and _is_false_like(metadata["turn_off_message_logging"]): + if _opts_out_of_message_redaction(metadata): stripped.append(f"{slot_label}.turn_off_message_logging") metadata.pop("turn_off_message_logging", None) if stripped: diff --git a/litellm/proxy/management_endpoints/config_override_endpoints.py b/litellm/proxy/management_endpoints/config_override_endpoints.py index a9c3f2b953f..7bc6b63ccc2 100644 --- a/litellm/proxy/management_endpoints/config_override_endpoints.py +++ b/litellm/proxy/management_endpoints/config_override_endpoints.py @@ -311,6 +311,9 @@ def _clear_cyberark_state(proxy_config: "ProxyConfig") -> None: proxy_config._last_cyberark_config = None # pyright: ignore[reportPrivateUsage] # proxy-internal helper, mirrors hashicorp endpoint usage +_STORED_CONFIG_VALUE: Final = TypeAdapter(dict[str, object]) + + async def _persist_cyberark_config( prisma_client: "PrismaClient", proxy_config: "ProxyConfig", @@ -331,7 +334,7 @@ async def _persist_cyberark_config( }, }, ) - return safe_json_loads(config_value) + return _STORED_CONFIG_VALUE.validate_python(safe_json_loads(config_value)) # --- Hashicorp Vault endpoints --- diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 3caaf5ba1af..b1f0002e143 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -34,6 +34,8 @@ import json from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, cast +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_span import db_span @@ -57,6 +59,8 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # ValueClass = Literal["migrated", "legacy", "plaintext", "undecryptable", "not-a-string"] +_DECODED_JSON: Final = TypeAdapter(object) + @dataclass class LocationReport: @@ -233,7 +237,7 @@ async def _migrate_config_settings_row( settings = record.param_value if isinstance(settings, str): - settings = json.loads(settings) + settings = _DECODED_JSON.validate_python(json.loads(settings)) if not isinstance(settings, dict): return report @@ -286,7 +290,7 @@ async def _migrate_sso_config(prisma_client: object, dry_run: bool) -> LocationR settings = record.sso_settings if isinstance(settings, str): - settings = json.loads(settings) + settings = _DECODED_JSON.validate_python(json.loads(settings)) if not isinstance(settings, dict): return report @@ -389,7 +393,8 @@ async def _migrate_callback_vars_table( # Real run: re-encrypt the legacy ciphertext to AES via the proven # selective transforms and persist. Never drop a row on failure. try: - re_encrypted = encrypt_callback_vars(decrypt_callback_vars(metadata)) + decrypted_metadata = _DECODED_JSON.validate_python(decrypt_callback_vars(metadata)) + re_encrypted = _DECODED_JSON.validate_python(encrypt_callback_vars(decrypted_metadata)) except Exception as e: # pragma: no cover - defensive; never drop a row verbose_proxy_logger.warning( "Skipping %s row %s callback_vars (transform failed): %s", @@ -528,7 +533,7 @@ async def _scan_one_table( return report for row in rows or []: for col in json_columns: - raw = getattr(row, col, None) + raw: object = getattr(row, col, None) if raw is None: continue if db_attr == "litellm_mcpservertable" and col in ("static_headers", "env"): @@ -536,7 +541,7 @@ async def _scan_one_table( continue if isinstance(raw, str): try: - raw = json.loads(raw) + raw = _DECODED_JSON.validate_python(json.loads(raw)) except (ValueError, TypeError): pass for s in _iter_encrypted_strings(raw): @@ -561,7 +566,7 @@ async def _scan_config_env_vars(prisma_client: object) -> LocationReport: value = record.param_value if isinstance(value, str): try: - value = json.loads(value) + value = _DECODED_JSON.validate_python(json.loads(value)) except (ValueError, TypeError): value = {} for s in _iter_encrypted_strings(value): diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 048bc41496f..7877d887409 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -450,7 +450,7 @@ def _generate_budget_windows( def _effective_key_for_generate(data: GenerateKeyRequest, now: datetime) -> LiteLLM_VerificationToken: - requested: Final = data.model_dump(exclude_unset=True, exclude_none=True) + requested: Final = _as_object_dict(data.model_dump(exclude_unset=True, exclude_none=True)) metadata_fields: Final = MappingProxyType( {field: value for field, value in requested.items() if field in _KEY_METADATA_REQUEST_FIELDS} ) @@ -493,7 +493,7 @@ def _regenerate_request_as_update_request(key: str, data: RegenerateKeyRequest) changed_fields: Final = MappingProxyType( { field: value - for field, value in data.model_dump(exclude_unset=True).items() + for field, value in _as_object_dict(data.model_dump(exclude_unset=True)).items() if field in UpdateKeyRequest.model_fields and field != "key" and not (field in _EMPTY_DURATION_MEANS_UNCHANGED and value == "") diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 5f65bc498ad..923c82e5f63 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -61,6 +61,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME, MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, @@ -102,6 +103,10 @@ class _HasServerId(Protocol): server_id: str +def _session_cookie_claims(claims: Mapping[str, object]) -> Mapping[str, object]: + return claims + + def does_mcp_server_exist(mcp_server_records: Iterable[_HasServerId], mcp_server_id: str) -> bool: """ Check if the mcp server with the given id exists in the iterable of mcp servers. @@ -585,13 +590,13 @@ if MCP_AVAILABLE: if decrypted_json is None: return None try: - loaded: Final = json.loads(decrypted_json) + loaded: Final = _CACHED_VALUE.validate_python(json.loads(decrypted_json)) except Exception as e: verbose_proxy_logger.debug("Invalid decrypted temporary MCP payload in Redis cache: %s", e) return None - if not isinstance(loaded, dict): + payload_dict: Final = as_str_mapping(loaded) + if payload_dict is None: return None - payload_dict: Final[dict[str, object]] = loaded try: return MCPServer.model_validate(payload_dict) @@ -2194,8 +2199,9 @@ if MCP_AVAILABLE: # UI session cookies may omit exp; don't require it. options={"verify_exp": False, "verify_aud": False}, ) - if decoded.get("login_method") in ("sso", "username_password"): - cookie_key: Final[str] = decoded.get("key", "") + session_claims: Final = _session_cookie_claims(decoded) + if session_claims.get("login_method") in ("sso", "username_password"): + cookie_key: Final = session_claims.get("key", "") if cookie_key: api_key = f"Bearer {cookie_key}" except _jwt.InvalidTokenError: @@ -3369,10 +3375,12 @@ if MCP_AVAILABLE: "openapi_registry.json", ) + _OPENAPI_REGISTRY_ADAPTER: Final = TypeAdapter(dict[str, object]) + @functools.lru_cache(maxsize=1) def _load_openapi_registry() -> dict[str, object]: with open(_OPENAPI_REGISTRY_PATH, "r") as f: - data: Final[dict[str, object]] = json.load(f) + data: Final = _OPENAPI_REGISTRY_ADAPTER.validate_python(json.load(f)) return data @router.get( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 9199472d20d..36071bfe51a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1092,14 +1092,9 @@ def update_db_model( _cost_map_entry(db_model, incoming_model_info), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) + stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) merged_model_info.update( - MappingProxyType( - dict( - (k, v) - for k, v in without_server_derived_pricing(incoming_model_info).items() - if k not in echoed_fields - ) - ) + MappingProxyType(dict((k, v) for k, v in stored_model_info_fields.items() if k not in echoed_fields)) ) for k in echoed_fields: if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]: diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index aa483eb2f6e..64566221d4d 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -129,8 +129,7 @@ async def get_router_settings( if field.field_name == "routing_groups": continue if hasattr(llm_router, field.field_name): - value = getattr(llm_router, field.field_name) - current_values[field.field_name] = value + current_values[field.field_name] = getattr(llm_router, field.field_name, None) # Merge with config values (config takes precedence) current_values.update(router_settings_from_config) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index c2d7033c06c..d1e1c4bd25e 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -8,13 +8,16 @@ import asyncio import copy import json import traceback +from collections.abc import Mapping from datetime import datetime, timezone from typing import Annotated, Final from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ( AddTeamCallback, LiteLLM_AuditLogs, @@ -62,6 +65,11 @@ router: Final = APIRouter() _CALLBACK_VARS_REDACTED: Final = "***REDACTED***" +_DECODED_JSON: Final = TypeAdapter(object) + + +def _decrypted_metadata(metadata: object) -> Mapping[str, object]: + return as_str_mapping(_DECODED_JSON.validate_python(decrypt_callback_vars(metadata))) or {} def _callback_config_error(message: str) -> HTTPException: @@ -147,7 +155,7 @@ def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: if not isinstance(team_metadata, dict): return TeamCallbackMetadata() - decrypted: Final = decrypt_callback_vars(team_metadata) + decrypted: Final = _decrypted_metadata(team_metadata) logging_entries: Final = decrypted.get("logging") if logging_entries is not None: @@ -357,7 +365,7 @@ async def add_team_callbacks( # Decrypted, because the checks compare the incoming values against # the stored ones and the credentials are encrypted at rest. - decrypted_logging: Final = decrypt_callback_vars(team_metadata).get("logging") + decrypted_logging: Final = _decrypted_metadata(team_metadata).get("logging") stored_callbacks: Final = stored_callback_entries(decrypted_logging) stored_entries: Final = decrypted_logging if isinstance(decrypted_logging, list) else () stored_entry_vars: Final = [entry.get("callback_vars") or {} for entry in stored_entries] @@ -411,8 +419,8 @@ async def add_team_callbacks( team_callback_settings.append(data.model_dump()) team_metadata["logging"] = team_callback_settings - team_metadata = encrypt_callback_vars(team_metadata) - team_metadata_json: Final = json.dumps(team_metadata) # update team_metadata + encrypted_metadata: Final = _DECODED_JSON.validate_python(encrypt_callback_vars(team_metadata)) + team_metadata_json: Final = json.dumps(encrypted_metadata) new_team_row: Final = await TeamRepository(prisma_client).table.update( where={"team_id": team_id}, @@ -436,7 +444,7 @@ async def add_team_callbacks( await _emit_team_callback_audit_log( team_id=team_id, before_metadata=before_metadata, - after_metadata=team_metadata, + after_metadata=encrypted_metadata, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, ) @@ -541,7 +549,7 @@ async def delete_team_callback( raise _callback_error(404, f"callback_name = {callback_name} is not registered for team_id = {team_id}.") updated_metadata: Final = {**team_metadata, "logging": remaining_callbacks} - encrypted_metadata: Final[object] = encrypt_callback_vars(updated_metadata) + encrypted_metadata: Final = _DECODED_JSON.validate_python(encrypt_callback_vars(updated_metadata)) team_metadata_json: Final = json.dumps(encrypted_metadata) updated_team: Final = await TeamRepository(prisma_client).table.update( @@ -678,7 +686,7 @@ async def disable_team_logging( # _get_dynamic_logging_metadata stops at metadata["logging"], where the API # and Admin UI register callbacks, without ever reading callback_settings. team_metadata["logging"] = [] - encrypted_metadata: Final[object] = encrypt_callback_vars(team_metadata) + encrypted_metadata: Final = _DECODED_JSON.validate_python(encrypt_callback_vars(team_metadata)) team_metadata_json: Final = json.dumps(encrypted_metadata) # Update team in database diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index ba85ea590b5..16e62513b7f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -19,10 +19,9 @@ import secrets from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from html import escape -from types import MappingProxyType +from types import MappingProxyType, TracebackType from typing import ( TYPE_CHECKING, - Any, Final, Literal, NoReturn, @@ -63,6 +62,7 @@ from litellm.constants import ( MICROSOFT_USER_ID_ATTRIBUTE, MICROSOFT_USER_LAST_NAME_ATTRIBUTE, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -193,6 +193,19 @@ class _UserMetadataRow(Protocol): def metadata(self) -> Mapping[str, object] | None: ... +class _GenericSSOProvider(Protocol): + def __enter__(self) -> object: ... + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: ... + + async def get_login_redirect(self, *, state: str | None = None) -> RedirectResponse: ... + + def _user_meta_db(repo: UserRepository) -> "TableActions[_UserMetadataRow]": return cast( # cast-ok: prisma types Json columns as str; the client hands back the deserialized value "TableActions[_UserMetadataRow]", repo.table @@ -219,6 +232,7 @@ def _team_detail_db(repo: TeamRepository) -> "TableActions[_TeamDetailRow]": _SSO_TOKEN_CLAIMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class _TeamRowGrants(LiteLLMBaseModel): @@ -267,10 +281,6 @@ class _ServicePrincipalPage(Protocol): def get(self, key: Literal["@odata.nextLink"]) -> str | None: ... -def _as_object(value: object) -> object: - return value - - def _hash_cli_sso_secret(secret: str) -> str: return hashlib.sha256(secret.encode("utf-8")).hexdigest() @@ -3185,7 +3195,7 @@ class SSOAuthenticationHandler: @staticmethod @with_service_target(SSO_SESSIONS_TARGET) async def get_generic_sso_redirect_response( - generic_sso: Any, + generic_sso: _GenericSSOProvider, state: str | None = None, generic_authorization_endpoint: str | None = None, request: Request | None = None, @@ -3214,13 +3224,9 @@ class SSOAuthenticationHandler: ) # Separate PKCE params from state params (fastapi-sso doesn't accept code_challenge) - pkce_params: Final = {} - state_only_params: Final = {} - for key, value in redirect_params.items(): - if key in ("code_challenge", "code_challenge_method"): - pkce_params[key] = value - else: - state_only_params[key] = value + pkce_keys: Final = ("code_challenge", "code_challenge_method") + pkce_params: Final = {key: value for key, value in redirect_params.items() if key in pkce_keys} + state_only_params: Final = {key: value for key, value in redirect_params.items() if key not in pkce_keys} # Get the redirect response from fastapi-sso with only state param redirect_response: Final = await generic_sso.get_login_redirect(**state_only_params) @@ -3278,7 +3284,7 @@ class SSOAuthenticationHandler: # concurrent SSO sessions — one PKCE, one plain — cannot # overwrite each other's state cookie. state_value: Final = redirect_params.get("state") - if state_value and redirect_response is not None: + if state_value: # Production-safe default: require HTTPS for the # CSRF-protection cookie unless we can prove the # incoming request is HTTP (local dev). Without @@ -3913,18 +3919,19 @@ class SSOAuthenticationHandler: cache_key: Final = f"pkce_verifier:{state}" if redis_usage_cache is not None: - cached_data = await redis_usage_cache.async_get_cache(key=cache_key) + cached_data = _DECODED_JSON.validate_python(await redis_usage_cache.async_get_cache(key=cache_key)) else: - cached_data = await user_api_key_cache.async_get_cache(key=cache_key) + cached_data = _DECODED_JSON.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) code_verifier = None # Track why code_verifier is absent for accurate strict-mode diagnostics. _empty_value_in_dict = False # dict format correct but value is empty/null if cached_data: + cached_mapping: Final = as_str_mapping(cached_data) # Extract code_verifier from dict (stored as dict for JSON serialization) - if isinstance(cached_data, dict) and "code_verifier" in cached_data: - code_verifier = cached_data["code_verifier"] + if isinstance(cached_mapping, dict) and "code_verifier" in cached_mapping: + code_verifier = cached_mapping["code_verifier"] if not code_verifier: # Dict format is correct but value is empty or null. This is # a distinct case from an unrecognized format — the entry exists @@ -4107,7 +4114,7 @@ class SSOAuthenticationHandler: access_token string. Raises ProxyException on any validation failure. """ try: - token_response_raw: Final[object] = _as_object(response.json()) + token_response_raw: Final = _DECODED_JSON.validate_python(response.json()) except Exception as json_err: verbose_proxy_logger.error( "Failed to parse token response as JSON: %s. Body: %s", diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 86545e6e064..3468396c6f6 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -15,6 +15,7 @@ from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ObjectPermissionDict, SpecialMCPServerName, SpecialMCPServerNames from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast @@ -129,6 +130,15 @@ async def prepare_object_permission_upsert( return ObjectPermissionUpsert(object_permission_id=object_permission_id, record=record) +_DECODED_JSON: Final = TypeAdapter(object) + + +def _object_permission_mapping(raw: object) -> Mapping[str, object]: + decoded: Final = _DECODED_JSON.validate_python(json.loads(raw)) if isinstance(raw, str) else raw + mapping: Final = as_str_mapping(decoded) + return mapping if mapping is not None and isinstance(decoded, dict) else {} + + async def handle_update_object_permission_common( data_json: dict, existing_object_permission_id: str | None, @@ -158,15 +168,12 @@ async def handle_update_object_permission_common( if prisma_client is None: raise ValueError("Prisma client not found") - new_object_permission: dict | str | None = data_json.pop("object_permission", None) - if new_object_permission is None: + raw_object_permission: Final[object] = data_json.pop("object_permission", None) + if raw_object_permission is None: return None - if isinstance(new_object_permission, str): - new_object_permission = json.loads(new_object_permission) - upsert: Final = await prepare_object_permission_upsert( - new_object_permission=new_object_permission if isinstance(new_object_permission, dict) else {}, + new_object_permission=_object_permission_mapping(raw_object_permission), existing_object_permission_id=existing_object_permission_id, prisma_client=prisma_client, ) diff --git a/litellm/proxy/middleware/in_flight_requests_middleware.py b/litellm/proxy/middleware/in_flight_requests_middleware.py index 2430f2fb081..7820805d424 100644 --- a/litellm/proxy/middleware/in_flight_requests_middleware.py +++ b/litellm/proxy/middleware/in_flight_requests_middleware.py @@ -6,11 +6,17 @@ Prometheus gauge `litellm_in_flight_requests`. """ import os -from typing import Any, Final +from typing import Final, Protocol from starlette.types import ASGIApp, Receive, Scope, Send +class _InFlightGauge(Protocol): + def inc(self) -> None: ... + + def dec(self) -> None: ... + + class InFlightRequestsMiddleware: """ ASGI middleware that increments a counter when a request arrives and @@ -27,7 +33,7 @@ class InFlightRequestsMiddleware: """ _in_flight: int = 0 - _gauge: Any | None = None + _gauge: _InFlightGauge | None = None _gauge_init_attempted: bool = False def __init__(self, app: ASGIApp) -> None: @@ -55,7 +61,7 @@ class InFlightRequestsMiddleware: return InFlightRequestsMiddleware._in_flight @staticmethod - def _get_gauge() -> Any | None: + def _get_gauge() -> _InFlightGauge | None: if InFlightRequestsMiddleware._gauge_init_attempted: return InFlightRequestsMiddleware._gauge InFlightRequestsMiddleware._gauge_init_attempted = True diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 6dd61a7bf14..48306cc8d7a 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -16,6 +16,8 @@ from typing import ( runtime_checkable, ) +from pydantic import TypeAdapter + from litellm.batches.batch_utils import batch_cost_is_final from litellm.constants import MAX_FILE_LIST_LIMIT from litellm.proxy._types import ProxyException @@ -885,7 +887,10 @@ def is_gemini_supported_mime_type(mime_type: str) -> bool: ) -def get_content_type_from_file_object(file_object: dict | None) -> str: +_DECODED_JSON: Final = TypeAdapter(object) + + +def get_content_type_from_file_object(file_object: object) -> str: """ Determine content type from file object (from database or API response). @@ -901,15 +906,19 @@ def get_content_type_from_file_object(file_object: dict | None) -> str: if not file_object: return "application/octet-stream" - # Handle JSON string - if isinstance(file_object, str): - import json + if not isinstance(file_object, str): + return _content_type_from_decoded_file_object(file_object) - try: - file_object = json.loads(file_object) - except json.JSONDecodeError: - return "application/octet-stream" + import json + try: + decoded_file_object: Final = _DECODED_JSON.validate_python(json.loads(file_object)) + except json.JSONDecodeError: + return "application/octet-stream" + return _content_type_from_decoded_file_object(decoded_file_object) + + +def _content_type_from_decoded_file_object(file_object: object) -> str: if not isinstance(file_object, dict): return "application/octet-stream" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 89e0f01076e..5d825099ceb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -145,6 +145,7 @@ else: vertex_llm_base: Final = VertexBase() router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) default_vertex_config: Final = None passthrough_endpoint_router: Final = PassthroughEndpointRouter() @@ -3549,7 +3550,7 @@ async def _tinyfish_json_object_field_names(request: Request) -> frozenset[str] if not raw_body: return frozenset() try: - parsed: Final[object] = json.loads(raw_body) # any-ok: json.loads -> Any + parsed: Final = _DECODED_JSON.validate_python(json.loads(raw_body)) except (json.JSONDecodeError, UnicodeDecodeError): return None return frozenset(parsed) if isinstance(parsed, dict) else None diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index 49679fe838b..0093c6f386d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -1,5 +1,5 @@ import re -from collections.abc import Callable +from collections.abc import Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final @@ -9,16 +9,13 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.gemini.videos.transformation import GeminiVideoConfig -from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - ModelResponseIterator as GeminiModelResponseIterator, -) from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + GeminiSseChunkParser, VertexPassthroughLoggingHandler, ) from litellm.types.utils import ( ModelResponse, - ModelResponseStream, TextCompletionResponse, ) @@ -184,33 +181,25 @@ class GeminiPassthroughLoggingHandler: @staticmethod def _build_complete_streaming_response( - all_chunks: list[str], + all_chunks: Sequence[str], litellm_logging_obj: LiteLLMLoggingObj, model: str, url_route: str, ) -> ModelResponse | TextCompletionResponse | None: - parsed_chunks = [] if "generateContent" in url_route or "streamGenerateContent" in url_route: - gemini_iterator: Final[Any] = GeminiModelResponseIterator( + gemini_iterator: Final = GeminiSseChunkParser( streaming_response=None, sync_stream=False, logging_obj=litellm_logging_obj, ) - chunk_parsing_logic: Final[Callable[[str], ModelResponseStream | None]] = ( - gemini_iterator._common_chunk_parsing_logic - ) - parsed_chunks = [chunk_parsing_logic(chunk) for chunk in all_chunks] + parsed_chunks: Final = [gemini_iterator.parse_sse_chunk(chunk) for chunk in all_chunks] else: return None if len(parsed_chunks) == 0: return None - all_openai_chunks: Final = [] - for parsed_chunk in parsed_chunks: - if parsed_chunk is None: - continue - all_openai_chunks.append(parsed_chunk) + all_openai_chunks: Final = [parsed_chunk for parsed_chunk in parsed_chunks if parsed_chunk is not None] complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index cfa569b5219..c3af6f534c2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -1,8 +1,8 @@ import asyncio import re -from collections.abc import Callable, Mapping +from collections.abc import Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast from urllib.parse import urlparse import httpx @@ -56,6 +56,7 @@ else: _VERTEX_INTERACTIONS_PATH: Final = re.compile(r"/projects/[^/]+/locations/[^/]+/interactions/?$") _INTERACTIONS_RESPONSE_BODY: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) @@ -72,6 +73,19 @@ def _interactions_model( return None +class GeminiSseChunkParser(VertexModelResponseIterator): + def parse_sse_chunk(self, chunk: str) -> ModelResponseStream | None: + return self._common_chunk_parsing_logic(chunk=chunk) + + +class _DictChunkParser(Protocol): + def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: ... + + +def _dict_chunk_parser(iterator: _DictChunkParser) -> _DictChunkParser: + return iterator + + class VertexPassthroughLoggingHandler: @staticmethod def is_interactions_route(url_route: str) -> bool: @@ -245,7 +259,7 @@ class VertexPassthroughLoggingHandler: url_route ) - _json_response: Final = httpx_response.json() + _json_response: Final = _DECODED_JSON.validate_python(httpx_response.json()) litellm_prediction_response = ModelResponse() @@ -620,45 +634,39 @@ class VertexPassthroughLoggingHandler: @staticmethod def _build_complete_streaming_response( - all_chunks: list[str], + all_chunks: Sequence[str], litellm_logging_obj: LiteLLMLoggingObj, model: str, url_route: str, ) -> ModelResponse | TextCompletionResponse | None: - parsed_chunks = [] if "generateContent" in url_route or "streamGenerateContent" in url_route: - vertex_iterator: Any = VertexModelResponseIterator( + vertex_iterator: Final = GeminiSseChunkParser( streaming_response=None, sync_stream=False, logging_obj=litellm_logging_obj, ) - chunk_parsing_logic: Callable[..., ModelResponseStream | None] = vertex_iterator._common_chunk_parsing_logic - parsed_chunks = [chunk_parsing_logic(chunk) for chunk in all_chunks] + parsed_chunks = [vertex_iterator.parse_sse_chunk(chunk) for chunk in all_chunks] elif "rawPredict" in url_route or "streamRawPredict" in url_route: from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.llms.base_llm.base_model_iterator import ( BaseModelResponseIterator, ) - vertex_iterator = ModelResponseIterator( - streaming_response=None, - sync_stream=False, + anthropic_iterator: Final = _dict_chunk_parser( + ModelResponseIterator( + streaming_response=None, + sync_stream=False, + ) ) - chunk_parsing_logic = vertex_iterator.chunk_parser - for chunk in all_chunks: - dict_chunk = BaseModelResponseIterator.string_to_dict_parser(chunk) - if dict_chunk is None: - continue - parsed_chunks.append(chunk_parsing_logic(dict_chunk)) + dict_chunks: Final = (BaseModelResponseIterator.string_to_dict_parser(chunk) for chunk in all_chunks) + parsed_chunks = [ + anthropic_iterator.chunk_parser(dict_chunk) for dict_chunk in dict_chunks if dict_chunk is not None + ] else: return None if len(parsed_chunks) == 0: return None - all_openai_chunks: Final = [] - for parsed_chunk in parsed_chunks: - if parsed_chunk is None: - continue - all_openai_chunks.append(parsed_chunk) + all_openai_chunks: Final = [parsed_chunk for parsed_chunk in parsed_chunks if parsed_chunk is not None] complete_streaming_response: Final = litellm.stream_chunk_builder(chunks=all_openai_chunks) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1663907b556..b1c448b023b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -159,6 +159,7 @@ router: Final = APIRouter() pass_through_endpoint_logging: Final = PassThroughEndpointLogging() _METADATA_KEYS: Final = frozenset(("litellm_metadata", "metadata")) +_DECODED_JSON: Final = TypeAdapter(object) _KEPT_OUT_OF_LITELLM_PARAMS: Final = _METADATA_KEYS | frozenset(wire_names(ProxyRequestState)) # Global registry to track registered pass-through routes and prevent memory leaks @@ -1167,7 +1168,7 @@ async def pass_through_request( # parsed dict (hooks mutate it, breaking the signature / Content-Length). # Tolerate request objects without `state` (test fixtures) and only honor # values httpx accepts for `content=`. - _request_state: Final = getattr(request, "state", None) + _request_state: Final[object] = getattr(request, "state", None) state_raw_body: str | bytes | None = ( getattr(_request_state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, None) if _request_state is not None @@ -2322,7 +2323,7 @@ def _rewrite_vertex_live_setup_model(text_data: str, setup_model_rewriter: Calla if setup_model_rewriter is None: return text_data try: - message: Final[object] = json.loads(text_data) + message: Final[object] = _DECODED_JSON.validate_python(json.loads(text_data)) except json.JSONDecodeError: return text_data if not isinstance(message, dict): @@ -2357,7 +2358,9 @@ def _resolved_vertex_live_setup( def _json_object_frame(frame: str | bytes) -> dict[str, object] | None: try: - decoded: Final[object] = json.loads(frame if isinstance(frame, str) else frame.decode("utf-8")) + decoded: Final[object] = _DECODED_JSON.validate_python( + json.loads(frame if isinstance(frame, str) else frame.decode("utf-8")) + ) except (json.JSONDecodeError, UnicodeDecodeError): return None return decoded if isinstance(decoded, dict) else None @@ -2586,7 +2589,7 @@ async def websocket_passthrough_request( endpoint, ) try: - client_message = json.loads(text_data) + client_message = _DECODED_JSON.validate_python(json.loads(text_data)) if isinstance(client_message, dict) and "setup" in client_message: setup_data = client_message["setup"] verbose_proxy_logger.debug( @@ -3970,7 +3973,7 @@ async def create_pass_through_endpoints( if data_dict.get("id") is None: data_dict["id"] = str(uuid.uuid4()) - if response.field_value is None: + if _config_field_endpoints(response) is None: response.field_value = [data_dict] elif isinstance(response.field_value, list): response.field_value.append(data_dict) @@ -4060,7 +4063,7 @@ async def delete_pass_through_endpoints( ## Update field by removing endpoint pass_through_endpoint_data: Final[list | None] = _config_field_endpoints(response) - if response.field_value is None or pass_through_endpoint_data is None: + if pass_through_endpoint_data is None: raise HTTPException( status_code=400, detail={"error": "There are no pass-through endpoints setup."}, diff --git a/litellm/proxy/policy_engine/init_policies.py b/litellm/proxy/policy_engine/init_policies.py index b6e84109222..d75e651e6c7 100644 --- a/litellm/proxy/policy_engine/init_policies.py +++ b/litellm/proxy/policy_engine/init_policies.py @@ -13,7 +13,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.proxy.policy_engine.policy_validator import PolicyValidator -from litellm.types.proxy.policy_engine import PolicyValidationResponse +from litellm.types.proxy.policy_engine import Policy, PolicyValidationResponse, ResolvedPolicy if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -235,29 +235,15 @@ def get_policies_summary() -> dict[str, object]: resolved: Final = PolicyResolver.get_all_resolved_policies() - summary: Final[dict[str, Any]] = { + return { "initialized": True, "policy_count": len(resolved), "attachment_count": len(attachment_registry.get_all_attachments()), - "policies": {}, - "attachments": [], - } - - for policy_name, resolved_policy in resolved.items(): - policy = policy_registry.get_policy(policy_name) - summary["policies"][policy_name] = { - "inherit": policy.inherit if policy else None, - "description": policy.description if policy else None, - "guardrails_add": policy.guardrails.get_add() if policy else [], - "guardrails_remove": policy.guardrails.get_remove() if policy else [], - "condition": (policy.condition.model_dump() if policy and policy.condition else None), - "resolved_guardrails": resolved_policy.guardrails, - "inheritance_chain": resolved_policy.inheritance_chain, - } - - # Add attachment info - for attachment in attachment_registry.get_all_attachments(): - summary["attachments"].append( + "policies": { + policy_name: _policy_summary_entry(policy_registry.get_policy(policy_name), resolved_policy) + for policy_name, resolved_policy in resolved.items() + }, + "attachments": [ { "policy": attachment.policy, "scope": attachment.scope, @@ -265,6 +251,18 @@ def get_policies_summary() -> dict[str, object]: "keys": attachment.keys, "models": attachment.models, } - ) + for attachment in attachment_registry.get_all_attachments() + ], + } - return summary + +def _policy_summary_entry(policy: Policy | None, resolved_policy: ResolvedPolicy) -> Mapping[str, object]: + return { + "inherit": policy.inherit if policy else None, + "description": policy.description if policy else None, + "guardrails_add": policy.guardrails.get_add() if policy else [], + "guardrails_remove": policy.guardrails.get_remove() if policy else [], + "condition": (policy.condition.model_dump() if policy and policy.condition else None), + "resolved_guardrails": resolved_policy.guardrails, + "inheritance_chain": resolved_policy.inheritance_chain, + } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ec65fbdc0fc..fa2e7db5633 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -22,6 +22,7 @@ from collections.abc import ( Awaitable, Callable, Collection, + Iterable, Mapping, MutableMapping, Sequence, @@ -32,7 +33,7 @@ from collections.abc import ( from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain -from types import MappingProxyType, UnionType +from types import MappingProxyType, TracebackType, UnionType from typing import ( TYPE_CHECKING, Annotated, @@ -43,6 +44,7 @@ from typing import ( Optional, Protocol, TypeAlias, + TypeVar, Union, cast, get_args, @@ -323,6 +325,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( ) from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, drop_params_flag, get_litellm_metadata_from_kwargs, get_parent_otel_span_from_kwargs, @@ -1270,6 +1273,7 @@ _LOGIN_REQUEST_BODY: Final = TypeAdapter(_LoginRequestBody) _LOGIN_EXCHANGE_REQUEST_BODY: Final = TypeAdapter(_LoginExchangeRequestBody) _ANTHROPIC_BETA_HEADERS_RELOAD_CONFIG: Final = TypeAdapter(_AnthropicBetaHeadersReloadConfig) _WEBSEARCH_INTERCEPTION_LITELLM_SETTINGS: Final = TypeAdapter(_WebsearchInterceptionLitellmSettings) +_UNTYPED_VALUE: Final = TypeAdapter(object) class _AiohttpConnectorKwargs(TypedDict, total=False): @@ -1420,9 +1424,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState await initialize_from_worker_config(worker_config) else: # if not, assume it's a json string - worker_config = json.loads(worker_config) - if isinstance(worker_config, dict): - await initialize_from_worker_config(worker_config) + decoded_worker_config: Final = as_str_mapping(_UNTYPED_VALUE.validate_python(json.loads(worker_config))) + if decoded_worker_config is not None: + await initialize_from_worker_config(decoded_worker_config) enforce_master_key_boot_verdict( await with_stored_secrets_counted( @@ -2132,6 +2136,11 @@ class _ConfigParamRow(Protocol): param_value: Mapping[str, JsonValue] | None +class _ConfigParamValueRow(Protocol): + @property + def param_value(self) -> object: ... + + class _ConfigOverridesRow(Protocol): config_value: Mapping[str, JsonValue] | None @@ -2159,6 +2168,25 @@ class _UserTeamsRow(Protocol): def teams(self) -> Sequence[str]: ... +class _InvitationAcceptTx(Protocol): + @property + def litellm_invitationlink(self) -> TableActions[_InvitationLinkRow]: ... + + @property + def litellm_usertable(self) -> TableActions[_UserTableRow]: ... + + +class _InvitationAcceptTxManager(Protocol): + async def __aenter__(self) -> _InvitationAcceptTx: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + _ProxyModelRow: TypeAlias = "prisma_models.LiteLLM_ProxyModelTable" @@ -2189,6 +2217,13 @@ class _ExceptionRow(TypedDict, total=False): exception_counts: Mapping[str, int] +_RawQueryRowT = TypeVar("_RawQueryRowT") + + +async def _query_raw_rows(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RawQueryRowT] | None: + return await prisma_client.db.query_raw(sql_query, *args) + + @app.exception_handler(RequestValidationError) async def otel_request_validation_exception_handler(request: Request, exc: RequestValidationError): public_errors: Final = public_validation_errors(exc.errors()) @@ -3283,6 +3318,15 @@ async def increment_spend_counters( ) +def _cached_budget_limits(cached: object) -> object: + from_attribute: Final[object] = _UNTYPED_VALUE.validate_python(getattr(cached, "budget_limits", None)) + cached_mapping: Final = as_str_mapping(cached) + stored: Final[object] = from_attribute or ( + cached_mapping.get("budget_limits") if cached_mapping is not None and isinstance(cached, dict) else None + ) + return _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored + + async def _increment_spend_counters_batched( token: str | None, team_id: str | None, @@ -3368,14 +3412,12 @@ async def _increment_spend_counters_batched( ) return pending_window - key_obj: Final[object] = await user_api_key_cache.async_get_cache(key=hashed_token) + key_obj: Final[object] = _UNTYPED_VALUE.validate_python( + await user_api_key_cache.async_get_cache(key=hashed_token) + ) if key_obj is None: return key_pending - key_budget_limits = getattr(key_obj, "budget_limits", None) or ( - key_obj.get("budget_limits") if isinstance(key_obj, dict) else None - ) - if isinstance(key_budget_limits, str): - key_budget_limits = json.loads(key_budget_limits) + key_budget_limits: Final = _cached_budget_limits(key_obj) if not isinstance(key_budget_limits, list): return key_pending window_pending: Final = await asyncio.gather( @@ -3429,14 +3471,12 @@ async def _increment_spend_counters_batched( ) return pending_window - team_obj: Final[object] = await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}") + team_obj: Final[object] = _UNTYPED_VALUE.validate_python( + await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}") + ) if team_obj is None: return team_pending - team_budget_limits = getattr(team_obj, "budget_limits", None) or ( - team_obj.get("budget_limits") if isinstance(team_obj, dict) else None - ) - if isinstance(team_budget_limits, str): - team_budget_limits = json.loads(team_budget_limits) + team_budget_limits: Final = _cached_budget_limits(team_obj) if not isinstance(team_budget_limits, list): return team_pending window_pending: Final = await asyncio.gather( @@ -3837,7 +3877,7 @@ async def _get_source_cache_base_spend( ) -> float: source_cache_keys: Final = [source_cache_key] if isinstance(source_cache_key, str) else source_cache_key for cache_key in source_cache_keys: - source: object = await user_api_key_cache.async_get_cache(key=cache_key) + source: object = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) if source is None: continue if isinstance(source, dict): @@ -5090,7 +5130,7 @@ def build_redis_usage_cache(redis_params: Mapping[str, object]) -> RedisCache: if startup_nodes is None: env_cluster_nodes: Final = get_secret_str("REDIS_CLUSTER_NODES") if env_cluster_nodes is not None: - startup_nodes = json.loads(env_cluster_nodes) + startup_nodes = _UNTYPED_VALUE.validate_python(json.loads(env_cluster_nodes)) non_node_params: Final = {key: value for key, value in redis_params.items() if key != "startup_nodes"} if startup_nodes: return RedisClusterCache(startup_nodes=startup_nodes, **non_node_params) @@ -5358,7 +5398,9 @@ def resolve_classifier_plugin( sync `def classify` passes the runtime_checkable isinstance and would only fail on the first classified request, so reject it here where the error names the config key. """ - resolved: Final = get_instance_fn(value=plugin_path, config_file_path=config_file_path) + resolved: Final = _UNTYPED_VALUE.validate_python( + get_instance_fn(value=plugin_path, config_file_path=config_file_path) + ) if not isinstance(resolved, ClassifierPlugin) or not inspect.iscoroutinefunction( getattr(resolved, "classify", None) ): @@ -7905,17 +7947,17 @@ class ProxyConfig: if value is None: return None - parsed: dict | None = None + parsed: object = None if isinstance(value, dict): parsed = value elif isinstance(value, str): import json try: - parsed = yaml.safe_load(value) + parsed = _UNTYPED_VALUE.validate_python(yaml.safe_load(value)) except (yaml.YAMLError, json.JSONDecodeError): try: - parsed = json.loads(value) + parsed = _UNTYPED_VALUE.validate_python(json.loads(value)) except json.JSONDecodeError: pass @@ -8563,7 +8605,7 @@ class ProxyConfig: await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client) async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None: - config_record: Final = await get_config_param(prisma_client, "litellm_settings") + config_record: Final[_ConfigParamValueRow | None] = await get_config_param(prisma_client, "litellm_settings") if config_record is None or config_record.param_value is None: return raw_settings: Final = config_record.param_value @@ -8649,7 +8691,9 @@ class ProxyConfig: ) try: - config_record: Final = await get_config_param(prisma_client, "litellm_settings") + config_record: Final[_ConfigParamValueRow | None] = await get_config_param( + prisma_client, "litellm_settings" + ) if config_record is None or config_record.param_value is None: return @@ -8905,7 +8949,9 @@ class ProxyConfig: """ try: # Get anthropic beta headers reload configuration from database - config_record: Final = await get_config_param(prisma_client, "anthropic_beta_headers_reload_config") + config_record: Final[_ConfigParamValueRow | None] = await get_config_param( + prisma_client, "anthropic_beta_headers_reload_config" + ) if config_record is None or config_record.param_value is None: return # No configuration found, skip reload @@ -9351,7 +9397,7 @@ class ProxyConfig: if isinstance(credential, dict): credential_object = CredentialItem(**credential) elif isinstance(credential, BaseModel): - credential_object = CredentialItem(**credential.model_dump()) + credential_object = CredentialItem.model_validate(credential.model_dump()) decrypted_credential_values: Final = {} for k, v in credential_object.credential_values.items(): @@ -9687,7 +9733,7 @@ def _restamp_streaming_chunk_model( model_mismatch_logged: bool, fallback_was_attempted: bool = False, fallback_model_from_metadata: str | None = None, -) -> tuple[Any, bool]: +) -> tuple[object, bool]: if should_return_raw_model_name(request_data): return chunk, model_mismatch_logged @@ -9773,7 +9819,7 @@ def _fast_serialize_simple_model_response_stream( if getattr(choice, "logprobs", None) is not None or getattr(choice, "enhancements", None) is not None: return None - delta: Final = getattr(choice, "delta", None) + delta: Final[object] = getattr(choice, "delta", None) if delta is None: return None @@ -9850,10 +9896,14 @@ def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: return False if getattr(choice, "logprobs", None) is not None: return False - delta: Final = getattr(choice, "delta", None) + delta: Final = choice.delta if hasattr(choice, "delta") else None if delta is None: return True - return all(value is None for value in delta.model_dump().values()) + return _all_values_none(delta.model_dump().values()) + + +def _all_values_none(values: Iterable[object]) -> bool: + return all(value is None for value in values) async def _apply_streaming_chunk_hooks( @@ -9862,7 +9912,7 @@ async def _apply_streaming_chunk_hooks( user_api_key_dict: UserAPIKeyAuth, request_data: dict, str_so_far: str, -) -> tuple[Any, str]: +) -> tuple[object, str]: chunk = await proxy_logging_obj.async_post_call_streaming_hook( user_api_key_dict=user_api_key_dict, response=chunk, @@ -9877,7 +9927,7 @@ async def _apply_streaming_chunk_hooks( return chunk, str_so_far -def _format_streaming_sse_chunk(chunk: str | bytes) -> str | bytes: +def _format_streaming_sse_chunk(chunk: object) -> str | bytes: if isinstance(chunk, bytes): return b"data: " + chunk + b"\n\n" return f"data: {chunk}\n\n" @@ -9908,6 +9958,10 @@ _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +def _object_stream(stream: AsyncIterator[object]) -> AsyncIterator[object]: + return stream + + async def _iter_with_keepalive( aiter: AsyncIterator[object], resolve_keepalive_seconds: Callable[[object], float], @@ -10117,7 +10171,7 @@ async def async_data_generator( client_disconnected = False error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap() - stream_iterator: Final[AsyncIterator[object]] = ( + stream_iterator: Final = _object_stream( proxy_logging_obj.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=response, @@ -10178,7 +10232,7 @@ async def async_data_generator( if item is _STREAM_KEEPALIVE: yield ": ping\n\n" continue - chunk = cast(Any, item) # cast-ok: sentinel already handled above, item is a real chunk here + chunk = item if needs_per_chunk_hook: ### CALL HOOKS ### - modify outgoing data chunk, _str_so_far = await _apply_streaming_chunk_hooks( @@ -10228,7 +10282,7 @@ async def async_data_generator( continue if error_state is not None: - error_state.observe_chunk(cast(object, chunk)) # cast-ok: the helper validates legacy untyped chunks + error_state.observe_chunk(chunk) raw_passthrough = False if isinstance(chunk, BaseModel): chunk = _serialize_streaming_chunk(chunk) @@ -10807,15 +10861,17 @@ class ProxyStartupEvent: from prisma.errors import UniqueViolationError try: - config_table: Final = prisma_client.db.litellm_config + config_table: Final[TableActions[_ConfigParamValueRow]] = _config_param_table(prisma_client) row: Final = await config_table.find_unique(where={"param_name": TUNING_BASELINE_PARAM_NAME}) if row is not None: stored: Final = row.param_value - decoded: Final = json.loads(stored) if isinstance(stored, str) else stored + decoded: Final = as_str_mapping( + _UNTYPED_VALUE.validate_python(json.loads(stored)) if isinstance(stored, str) else stored + ) return MappingProxyType( { str(identity): str(fingerprint) - for identity, fingerprint in (decoded.items() if isinstance(decoded, Mapping) else ()) + for identity, fingerprint in (decoded.items() if decoded is not None else ()) } ) snapshot: Final = snapshot_tuning_baselines(deployments) @@ -10831,14 +10887,16 @@ class ProxyStartupEvent: except UniqueViolationError: competing_row: Final = await config_table.find_unique(where={"param_name": TUNING_BASELINE_PARAM_NAME}) competing_value: Final = None if competing_row is None else competing_row.param_value - competing_decoded: Final = ( - json.loads(competing_value) if isinstance(competing_value, str) else competing_value + competing_decoded: Final = as_str_mapping( + _UNTYPED_VALUE.validate_python(json.loads(competing_value)) + if isinstance(competing_value, str) + else competing_value ) return MappingProxyType( { str(identity): str(fingerprint) for identity, fingerprint in ( - competing_decoded.items() if isinstance(competing_decoded, Mapping) else () + competing_decoded.items() if competing_decoded is not None else () ) } ) @@ -11521,7 +11579,8 @@ class ProxyStartupEvent: async def check() -> None: settings: Final = cls._spend_capture_rate_check_settings(read_general_settings()) if settings is None: - for provider in get_args(SpendCaptureProvider): + providers: Final[tuple[SpendCaptureProvider, ...]] = get_args(SpendCaptureProvider) + for provider in providers: publish(provider, None) return await run_scheduled_spend_capture_rate_check( @@ -13038,6 +13097,10 @@ async def audio_speech( ) +def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in form_data.items() if key != "file"} + + @router.post( "/v1/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], @@ -13065,7 +13128,7 @@ async def audio_transcriptions( try: # Use orjson to parse JSON data, orjson speeds up requests significantly form_data: Final = await get_form_data(request) - data = {key: value for key, value in form_data.items() if key != "file"} | data + data = {**_form_fields_without_file(form_data), **data} # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -16101,8 +16164,8 @@ async def model_streaming_metrics( """ _all_api_bases: Final = set() - db_response: Final[Sequence[_TTFTRow] | None] = await prisma_client.db.query_raw( - sql_query, _selected_model_group, startTime, endTime + db_response: Final[Sequence[_TTFTRow] | None] = await _query_raw_rows( + prisma_client, sql_query, _selected_model_group, startTime, endTime ) _daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}} if db_response is not None: @@ -16225,8 +16288,8 @@ async def model_metrics( avg_latency_per_token DESC; """ _all_api_bases: Final = set() - db_response: Final[Sequence[_LatencyRow] | None] = await prisma_client.db.query_raw( - sql_query, _selected_model_group, startTime, endTime, api_key, customer + db_response: Final[Sequence[_LatencyRow] | None] = await _query_raw_rows( + prisma_client, sql_query, _selected_model_group, startTime, endTime, api_key, customer ) _daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}} @@ -16416,8 +16479,8 @@ async def model_metrics_exceptions( ORDER BY total_exceptions DESC LIMIT 200; """ - db_response: Final[Sequence[_ExceptionRow] | None] = await prisma_client.db.query_raw( - sql_query, startTime, endTime, _selected_model_group, api_key + db_response: Final[Sequence[_ExceptionRow] | None] = await _query_raw_rows( + prisma_client, sql_query, startTime, endTime, _selected_model_group, api_key ) response: Final[list[dict]] = [] exception_types: Final = set() @@ -17692,10 +17755,11 @@ async def login_v3_exchange(request: Request): ) cache_key: Final = f"login_code:{code}" - if redis_usage_cache is not None: - cached_data = await redis_usage_cache.async_get_cache(key=cache_key) - else: - cached_data = await user_api_key_cache.async_get_cache(key=cache_key) + cached_data: Final[object] = _UNTYPED_VALUE.validate_python( + await redis_usage_cache.async_get_cache(key=cache_key) + if redis_usage_cache is not None + else await user_api_key_cache.async_get_cache(key=cache_key) + ) if not cached_data or not isinstance(cached_data, dict): raise ProxyException( @@ -17998,7 +18062,8 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): await validate_password_not_breached(data.password, general_settings) hashed_pw: Final = hash_password(data.password) current_time = litellm.utils.get_utc_datetime() - async with prisma_client.db.tx() as tx: + tx_manager: Final[_InvitationAcceptTxManager] = prisma_client.db.tx() + async with tx_manager as tx: updated_count: Final = await tx.litellm_invitationlink.update_many( where={"id": data.invitation_link, "is_accepted": False}, data={ @@ -18824,8 +18889,9 @@ async def update_config_general_settings( detail={"error": CommonProxyErrors.not_allowed_access.value}, ) + requested_value: Final[object] = data.field_value if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS: - return await _persist_general_settings_ui_litellm_field(data.field_name, data.field_value, user_api_key_dict) + return await _persist_general_settings_ui_litellm_field(data.field_name, requested_value, user_api_key_dict) if data.field_name not in ConfigGeneralSettings.model_fields: raise HTTPException( @@ -18834,16 +18900,16 @@ async def update_config_general_settings( ) try: - ConfigGeneralSettings.model_validate({data.field_name: data.field_value}) + ConfigGeneralSettings.model_validate({data.field_name: requested_value}) except Exception: raise HTTPException( status_code=400, - detail={"error": f"Invalid type of field value={type(data.field_value)} passed in."}, + detail={"error": f"Invalid type of field value={type(requested_value)} passed in."}, ) if data.field_name == "alerting_args": try: - SlackAlertingArgs.model_validate(data.field_value) + SlackAlertingArgs.model_validate(requested_value) except ValidationError as e: errors: Final = "; ".join(f"{'.'.join(str(loc) for loc in err['loc'])}: {err['msg']}" for err in e.errors()) raise HTTPException( @@ -18868,10 +18934,10 @@ async def update_config_general_settings( proxy_config.reject_config_owned_writes( section_name="general_settings", - changed_keys={data.field_name: cast(JsonValue, data.field_value)}, # cast-ok: validated above + changed_keys={data.field_name: cast(JsonValue, requested_value)}, # cast-ok: validated above ) - field_value = data.field_value + field_value = requested_value if data.field_name == "plugins": field_value = _preserve_redacted_plugin_keys(field_value, general_settings.get("plugins")) @@ -19268,6 +19334,10 @@ async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key return {"message": f"Field {field_name} reset", "status": "success"} +def _model_annotations(model: type[BaseModel]) -> Mapping[str, object]: + return model.__annotations__ + + @router.get( "/config/list", tags=["config.yaml"], @@ -19343,7 +19413,7 @@ async def get_config_list( ), stored_in_db=None, ) - for sub_field, sub_field_type in pydantic_class.__annotations__.items() + for sub_field, sub_field_type in _model_annotations(pydantic_class).items() ] idx = 0 @@ -20835,7 +20905,7 @@ async def _is_mcp_access_group_cached(name: str) -> bool: ) cache_key: Final = f"mcp_access_group_exists:{name}" - cached: Final[object] = await user_api_key_cache.async_get_cache(key=cache_key) + cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) if cached is not None: return bool(cached) result: Final = bool(await MCPRequestHandler.get_mcp_servers_from_access_groups([name])) diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 07b1abb62e4..9c0f3c4b81c 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,6 +2,7 @@ import json import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -194,6 +195,10 @@ def _encode_realtime_token_payload( return json.dumps(payload, separators=(",", ":")) +def _header_value(headers: Mapping[str, str], name: str, default: str) -> str: + return headers.get(name, default) + + def _decode_realtime_token_payload( decrypted_value: str, ) -> dict[str, Any] | None: @@ -517,7 +522,7 @@ async def proxy_realtime_calls( return Response( content=upstream_resp.content, status_code=upstream_resp.status_code, - media_type=upstream_resp.headers.get("content-type", "application/sdp"), + media_type=_header_value(upstream_resp.headers, "content-type", "application/sdp"), ) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 7c52499bd6f..4bfc4433815 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -31,6 +31,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.openai.responses.guardrail_translation.handler import ( @@ -72,6 +73,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) +_DECODED_JSON: Final = TypeAdapter(object) _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg @@ -1643,7 +1645,7 @@ async def _read_ws_model_from_first_frame( return None try: - first_event: Final = json.loads(first_message) + first_event: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(first_message))) except json.JSONDecodeError: await websocket.send_text( json.dumps( @@ -1659,7 +1661,7 @@ async def _read_ws_model_from_first_frame( await websocket.close(code=1008, reason="Invalid JSON in first message") return None - if not isinstance(first_event, dict) or first_event.get("type") != "response.create": + if first_event is None or first_event.get("type") != "response.create": await websocket.send_text( json.dumps( { diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 3f82168d958..787f27f56c8 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -7,6 +7,8 @@ from collections.abc import Mapping from datetime import datetime, timezone from typing import Final +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid4 @@ -14,6 +16,8 @@ from litellm.caching.redis_cache import RedisCache from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus _RESPONSE_POLLING_TARGET: Final = "response_polling" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_LIST: Final = TypeAdapter(list[object]) class ResponsePollingHandler: @@ -153,7 +157,7 @@ class ResponsePollingHandler: return # Parse existing ResponsesAPIResponse from cache - state: Final = json.loads(cached_state) + state: Final = _JSON_OBJECT.validate_python(json.loads(cached_state)) # Update status (using OpenAI native status values) if status: @@ -215,7 +219,7 @@ class ResponsePollingHandler: ttl=self.ttl, ) - output_count: Final = len(state.get("output", [])) + output_count: Final = len(_JSON_LIST.validate_python(state.get("output", []))) verbose_proxy_logger.debug( "Updated polling state for %s: status=%s, output_items=%s", polling_id, state["status"], output_count ) @@ -230,7 +234,7 @@ class ResponsePollingHandler: cached_state: Final = await self.redis_cache.async_get_cache(cache_key) if cached_state: - return json.loads(cached_state) + return _JSON_OBJECT.validate_python(json.loads(cached_state)) return None diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 20cd62bf81b..31732f2dbc9 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -35,6 +35,7 @@ from litellm.constants import ( SPEND_CAPTURE_RATE_MAX_RANGE_DAYS, ) from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import * from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject from litellm.proxy.auth.authorization import ( @@ -84,6 +85,7 @@ else: router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 +_DECODED_JSON: Final = TypeAdapter(object) _SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" _SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" @@ -2494,7 +2496,7 @@ def _build_spend_log_search_condition( tags=["Budget & Spend Tracking"], dependencies=[Depends(user_api_key_auth)], responses={ - 200: {"model": dict[str, Any]}, + 200: {"model": dict[str, object]}, }, ) @router.get( @@ -3310,14 +3312,17 @@ def _hydrate_spend_log_metadata(rows: Sequence[Mapping[str, object]]) -> None: def _cold_storage_object_key_from_metadata( metadata: str | Mapping[str, object] | None, ) -> str | None: - if isinstance(metadata, str): - try: - metadata = json.loads(metadata) - except (json.JSONDecodeError, TypeError): - return None - if not isinstance(metadata, dict): + if not isinstance(metadata, str): + return _cold_storage_object_key(metadata if isinstance(metadata, dict) else None) + try: + decoded: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(metadata))) + except (json.JSONDecodeError, TypeError): return None - object_key: Final = metadata.get("cold_storage_object_key") + return _cold_storage_object_key(decoded) + + +def _cold_storage_object_key(metadata: Mapping[str, object] | None) -> str | None: + object_key: Final = metadata.get("cold_storage_object_key") if metadata is not None else None return object_key if isinstance(object_key, str) and object_key else None diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index cbfd249eb82..b327129793d 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -62,6 +62,8 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) + JsonSchemaItems: Final = TypedDict( "JsonSchemaItems", @@ -1267,7 +1269,7 @@ async def update_sso_settings( if existing_sso_record and existing_sso_record.sso_settings: stored = existing_sso_record.sso_settings if isinstance(stored, str): - stored = json.loads(stored) + stored = _DECODED_JSON.validate_python(json.loads(stored)) if isinstance(stored, dict): before_sso_data = ( # rebind-ok: pre-existing rebinding on a rename-only line proxy_config.decrypt_db_variables(stored) @@ -1720,7 +1722,7 @@ async def get_ui_settings_cached() -> dict[str, JsonValue]: from litellm.proxy.proxy_server import prisma_client, user_api_key_cache # 1. Try cache - cached: Final = await user_api_key_cache.async_get_cache(key=UI_SETTINGS_CACHE_KEY) + cached: Final = _DECODED_JSON.validate_python(await user_api_key_cache.async_get_cache(key=UI_SETTINGS_CACHE_KEY)) if cached is not None and isinstance(cached, dict): return cached @@ -1748,6 +1750,10 @@ async def get_ui_settings_cached() -> dict[str, JsonValue]: _UI_SETTINGS_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +def _json_settings(settings: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return settings + + def model_creation_disabled_for_internal_users(settings: Mapping[str, object]) -> bool: setting: Final = "disable_model_add_for_internal_users" return UISettings.model_validate({setting: settings.get(setting, False)}).disable_model_add_for_internal_users @@ -1954,7 +1960,7 @@ async def update_ui_settings( ) # Only include fields the caller actually sent (not Pydantic defaults). - settings_dict: Final[Mapping[str, JsonValue]] = settings.model_dump(exclude_unset=True) + settings_dict: Final = _json_settings(settings.model_dump(exclude_unset=True)) # Reject enterprise-only settings up front so the caller gets a clear # signal instead of a silent drop. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1860b1ab97b..699fde28d4d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -79,6 +79,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( + attribute_of, litellm_call_id_headers, openai_error_param, with_litellm_call_id, @@ -1239,6 +1240,10 @@ def _raise_malformed_listing_filter_return(error: MalformedListingFilterReturn) ) +def _proxy_hook_factory(hook_name: str) -> Callable[..., CustomLogger]: + return get_proxy_hook(hook_name) + + class ProxyLogging: """ Logging/Custom Handlers for proxy. @@ -1414,19 +1419,19 @@ class ProxyLogging: from litellm.proxy.proxy_server import prisma_client for hook in PROXY_HOOKS: - proxy_hook = get_proxy_hook(hook) + proxy_hook = _proxy_hook_factory(hook) expected_args = inspect.getfullargspec(proxy_hook).args if "prisma_client" in expected_args and prisma_client is None: verbose_proxy_logger.debug( "Skipping proxy hook %s: it requires a database and no prisma client is configured", hook ) continue - passed_in_args: dict[str, Any] = {} - if "internal_usage_cache" in expected_args: - passed_in_args["internal_usage_cache"] = self.internal_usage_cache - if "prisma_client" in expected_args: - passed_in_args["prisma_client"] = prisma_client - proxy_hook_obj = cast(CustomLogger, proxy_hook(**passed_in_args)) + available_args: Mapping[str, object] = { + "internal_usage_cache": self.internal_usage_cache, + "prisma_client": prisma_client, + } + passed_in_args = {name: value for name, value in available_args.items() if name in expected_args} + proxy_hook_obj = proxy_hook(**passed_in_args) litellm.logging_callback_manager.add_litellm_callback(proxy_hook_obj) self.proxy_hook_mapping[hook] = proxy_hook_obj @@ -6080,9 +6085,9 @@ class PrismaClient: if prisma_obj.is_connected() is not True: return 0 engine: Final = prisma_obj._engine # pyright: ignore[reportPrivateUsage] # Prisma engine internals - process: Final = getattr(engine, "process", None) if engine is not None else None + process: Final = attribute_of(engine, "process") if process is not None: - pid: Final[object] = process.pid + pid: Final = attribute_of(process, "pid") if isinstance(pid, int): return pid except (AttributeError, TypeError): diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 4a1f1dc19b8..6bfccd2426b 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -13,6 +13,7 @@ import json from typing import TYPE_CHECKING, Any, Final from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: @@ -88,6 +89,8 @@ _LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=fro _REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10 +_DECODED_JSON: Final = TypeAdapter(object) + def _redact_sensitive_litellm_params(litellm_params: object, _depth: int = 0) -> Any: """ @@ -115,7 +118,7 @@ def _redact_sensitive_litellm_params(litellm_params: object, _depth: int = 0) -> return None if isinstance(litellm_params, str): try: - parsed: Final[object] = json.loads(litellm_params) + parsed: Final = _DECODED_JSON.validate_python(json.loads(litellm_params)) except (TypeError, ValueError): return REDACTED_BY_LITELM_STRING return json.dumps(_redact_sensitive_litellm_params(parsed, _depth + 1)) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index b2ec2eda761..554f0f6ef53 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -7,8 +7,9 @@ from litellm.types.videos.utils import encode_character_id_with_provider def extract_model_from_target_model_names(target_model_names: Any) -> str | None: if isinstance(target_model_names, str): - target_model_names = [m.strip() for m in target_model_names.split(",") if m.strip()] - elif not isinstance(target_model_names, list): + model_names: Final = [m.strip() for m in target_model_names.split(",") if m.strip()] + return model_names[0] if model_names else None + if not isinstance(target_model_names, list): return None return target_model_names[0] if target_model_names else None diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 2d7b4c2f00f..02b95c80b60 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -269,7 +269,7 @@ async def arealtime_calls( openai_ephemeral_key: str, sdp_body: bytes, model: str | None = None, - session: dict[str, Any] | None = None, + session: dict[str, object] | None = None, timeout: float | None = None, **kwargs, ): diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index adbf0cdb72e..f7e2616324f 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -2,7 +2,7 @@ import asyncio import contextvars from collections.abc import Awaitable from functools import partial -from typing import Any, Final, Literal +from typing import Final, Literal import litellm from litellm._logging import verbose_logger @@ -31,7 +31,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() async def arerank( model: str, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: ( Literal["cohere", "together_ai", "deepinfra", "fireworks_ai", "voyage", "watsonx"] | None ) = None, @@ -93,7 +93,7 @@ async def arerank( def rerank( model: str, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: ( Literal[ "cohere", diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c90455e8e75..f9225dc1a79 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -296,9 +296,7 @@ async def aresponses_api_with_mcp( served_tools=original_mcp_tools, **kwargs, ) - await mcp_streaming_response._create_initial_response_iterator() - if mcp_streaming_response._initial_creation_error is not None: - raise mcp_streaming_response._initial_creation_error + await mcp_streaming_response.start_initial_response() return mcp_streaming_response # Determine if we should auto-execute tools @@ -1432,7 +1430,7 @@ def responses( current_reasoning: Final = cast( # cast-ok: prompt-managed reasoning arrives as a plain dict Reasoning | None, local_vars.get("reasoning") ) - reasoning_effort: Final = local_vars.get("reasoning_effort") + reasoning_effort: Final[object] = local_vars.get("reasoning_effort") request_reasoning: Final = ( LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort) if current_reasoning is None and reasoning_effort is not None diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 4dd0394005d..b711d051055 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Type from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam +from typing_extensions import TypedDict as ExtraItemsTypedDict +from typing_extensions import Unpack from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG @@ -43,6 +45,7 @@ if TYPE_CHECKING: from litellm.proxy._experimental.mcp_server.ui_session_utils import GrantedToolsetIds from litellm.proxy.utils import ProxyLogging + from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth else: MCPTool = Any @@ -58,6 +61,10 @@ class MCPToolResult(TypedDict): name: str | None +class _AresponsesOptions(ExtraItemsTypedDict, total=False, extra_items=object): + pass + + LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" @@ -1145,7 +1152,7 @@ class LiteLLM_Proxy_MCP_Handler: model: str, all_tools: Sequence[ResponsesToolParam] | None, response_id: str | None, - **call_params: Any, + **call_params: Unpack[_AresponsesOptions], ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: """Make follow-up response API call with tool results.""" return await aresponses( @@ -1195,7 +1202,7 @@ class LiteLLM_Proxy_MCP_Handler: tool_server_map: dict[str, str], served_tools: Sequence[MCPTool] | None = None, **kwargs, - ) -> Any: + ) -> "MCPEnhancedStreamingIterator": """ Create MCP enhanced streaming response that handles the full MCP workflow. @@ -1317,14 +1324,14 @@ class LiteLLM_Proxy_MCP_Handler: _create_tool_execution_events = create_tool_execution_events @staticmethod - def prepare_initial_call_params(call_params: Mapping[str, object], should_auto_execute: bool) -> dict[str, Any]: + def prepare_initial_call_params(call_params: Mapping[str, object], should_auto_execute: bool) -> _AresponsesOptions: """ Prepare call parameters for the initial LLM call. For auto-execute scenarios, we need to disable streaming for the initial call so we can process the tool calls before streaming the final response. """ - initial_params: Final = dict(call_params) + initial_params: Final = _AresponsesOptions(**call_params) if should_auto_execute: # Disable streaming for initial call when auto-executing tools @@ -1337,14 +1344,14 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def prepare_follow_up_call_params( call_params: Mapping[str, object], original_stream_setting: bool - ) -> dict[str, Any]: + ) -> _AresponsesOptions: """ Prepare call parameters for the follow-up LLM call after tool execution. Restores the original streaming setting and removes tool_choice since we're now providing tool results, not requesting tool calls. """ - follow_up_params: Final = dict(call_params) + follow_up_params: Final = _AresponsesOptions(**call_params) # Restore original streaming setting for follow-up call follow_up_params["stream"] = original_stream_setting diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 4ea114c1726..73cfba8dca4 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -730,6 +730,11 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Don't set phase to "finished" here — let __anext__ emit any # pre-generated MCP discovery events before ending the iteration. + async def start_initial_response(self) -> None: + await self._create_initial_response_iterator() + if self._initial_creation_error is not None: + raise self._initial_creation_error + async def _generate_tool_execution_events(self) -> None: """Generate tool execution events and execute tools""" if not self.collected_response: diff --git a/litellm/router.py b/litellm/router.py index 8fea9126c85..a549bb06e7d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -521,6 +521,7 @@ _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected" @@ -821,7 +822,8 @@ MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 def _retry_policy_ceiling(policy: RetryPolicy) -> int: """The most retries any error class under this policy can be granted.""" - return max((retries for retries in policy.model_dump().values() if isinstance(retries, int)), default=0) + retry_counts: Final[Iterable[object]] = policy.model_dump().values() + return max((retries for retries in retry_counts if isinstance(retries, int)), default=0) def _responses_stream_holds_event(item: object, held_event_count: int) -> bool: @@ -1534,7 +1536,7 @@ class Router: if not startup_nodes: _env_cluster_nodes: Final = get_secret("REDIS_CLUSTER_NODES") if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str): - startup_nodes = json.loads(_env_cluster_nodes) + startup_nodes = _OPAQUE_VALUE_ADAPTER.validate_python(json.loads(_env_cluster_nodes)) if startup_nodes: return RedisClusterCache(**{**cache_config, "startup_nodes": startup_nodes}) @@ -5416,17 +5418,19 @@ class Router: self._stamp_retry_skip_deployment_id(e, kwargs) raise e - async def _asearch_with_fallbacks(self, original_function: Callable, **kwargs): + async def _asearch_with_fallbacks(self, original_function: Callable, **kwargs) -> object: """ Helper function to make a search API call through the router with load balancing and fallbacks. Reuses the router's retry/fallback infrastructure. """ from litellm.router_utils.search_api_router import SearchAPIRouter - return await SearchAPIRouter.async_search_with_fallbacks( - router_instance=self, - original_function=original_function, - **kwargs, + return _OPAQUE_VALUE_ADAPTER.validate_python( + await SearchAPIRouter.async_search_with_fallbacks( + router_instance=self, + original_function=original_function, + **kwargs, + ) ) async def _asearch_with_fallbacks_helper(self, model: str, original_generic_function: Callable, **kwargs): @@ -6197,16 +6201,18 @@ class Router: ) else e ) - fallback_response: Final = await self.async_function_with_fallbacks_common_utils( - e=fallback_trigger, - disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), - fallbacks=fallbacks, - context_window_fallbacks=context_window_fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - model_group=model_group, - args=(), - kwargs=initial_kwargs, - include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True, + fallback_response: Final = _OPAQUE_VALUE_ADAPTER.validate_python( + await self.async_function_with_fallbacks_common_utils( + e=fallback_trigger, + disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), + fallbacks=fallbacks, + context_window_fallbacks=context_window_fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + model_group=model_group, + args=(), + kwargs=initial_kwargs, + include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True, + ) ) async for fallback_item in self._aanthropic_messages_yield_recovered(fallback_response, wrapper): yield fallback_item diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index f4860c4f097..c3a65d6aa91 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -18,7 +18,7 @@ import time from collections import OrderedDict from collections.abc import Mapping from dataclasses import asdict, dataclass -from typing import Any, Final, cast +from typing import Final, cast from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -180,8 +180,8 @@ class AdaptiveRouter: async def async_pre_routing_hook( self, model: str, - request_kwargs: dict[str, Any], - messages: list[dict[str, Any]] | None = None, + request_kwargs: dict[str, object], + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> PreRoutingHookResponse | None: diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 0b79cf8f17d..ee620790282 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -198,7 +198,7 @@ class AutoRouter(CustomLogger): self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> Optional["PreRoutingHookResponse"]: diff --git a/litellm/router_strategy/auto_router/litellm_encoder.py b/litellm/router_strategy/auto_router/litellm_encoder.py index caabbb3a342..b7430062f1c 100644 --- a/litellm/router_strategy/auto_router/litellm_encoder.py +++ b/litellm/router_strategy/auto_router/litellm_encoder.py @@ -94,14 +94,14 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin): ) return clamped - def __call__(self, docs: list[Any], **kwargs) -> list[list[float]]: + def __call__(self, docs: list[str], **kwargs) -> list[list[float]]: """Encode a list of text documents into embeddings using LiteLLM. :param docs: List of text documents to encode. :return: List of embeddings for each document.""" return self.encode_queries(docs, **kwargs) - async def acall(self, docs: list[Any], **kwargs) -> list[list[float]]: + async def acall(self, docs: list[str], **kwargs) -> list[list[float]]: """Encode a list of documents into embeddings using LiteLLM asynchronously. :param docs: List of documents to encode. diff --git a/litellm/router_strategy/complexity_router/capability_classifier.py b/litellm/router_strategy/complexity_router/capability_classifier.py index 717e8d389a3..783b90144cd 100644 --- a/litellm/router_strategy/complexity_router/capability_classifier.py +++ b/litellm/router_strategy/complexity_router/capability_classifier.py @@ -209,7 +209,7 @@ _JSON_DECODER: Final = json.JSONDecoder() def _complete_json_object_at(content: str, start: int) -> str | None: try: - _, end = _JSON_DECODER.raw_decode(content, start) + end: Final = _JSON_DECODER.raw_decode(content, start)[1] except (ValueError, RecursionError): return None return content[start:end] diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7ef31f7a0a2..1638f20c8e3 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1952,9 +1952,9 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None = None, - request_kwargs: dict[str, Any] | None = None, + request_kwargs: dict[str, object] | None = None, messages: Sequence[Mapping[str, object]] | None = None, - raw_messages: list[dict[str, Any]] | None = None, # mutable-ok: same shape _run_routing_plugins receives + raw_messages: list[dict[str, object]] | None = None, # mutable-ok: same shape _run_routing_plugins receives ) -> ClassificationOutcome: """ Classify a prompt by complexity, using the LLM classifier when configured. @@ -4331,7 +4331,7 @@ class ComplexityRouter(CustomLogger): self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> PreRoutingHookResponse | None: diff --git a/litellm/router_strategy/complexity_router/stall_detector.py b/litellm/router_strategy/complexity_router/stall_detector.py index 690603f05d5..2c9e0ed1b09 100644 --- a/litellm/router_strategy/complexity_router/stall_detector.py +++ b/litellm/router_strategy/complexity_router/stall_detector.py @@ -20,7 +20,10 @@ from collections.abc import Iterator, Mapping, Sequence from itertools import islice from typing import Final, NamedTuple +from pydantic import TypeAdapter + _ARGUMENTS_PARSE_FAILED: Final = object() +_DECODED_ARGUMENTS: Final = TypeAdapter(object) class _ToolCallEvent(NamedTuple): @@ -31,7 +34,7 @@ class _ToolCallEvent(NamedTuple): def _json_arguments(raw: str) -> object: try: - return json.loads(raw) + return _DECODED_ARGUMENTS.validate_python(json.loads(raw)) except (TypeError, ValueError): return _ARGUMENTS_PARSE_FAILED diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 2b37e7dfb40..527ea830d55 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -26,7 +26,7 @@ from .base_routing_strategy import BaseRoutingStrategy if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index e99d473d455..1d119375778 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -113,12 +113,12 @@ class QualityRouter(CustomLogger): return getattr(model_info, "litellm_routing_preferences", None) # Pydantic-object deployment. - model_info = getattr(deployment, "model_info", None) - if model_info is None: + deployment_model_info: Final[object] = getattr(deployment, "model_info", None) + if deployment_model_info is None: return None - if isinstance(model_info, dict): - return model_info.get("litellm_routing_preferences") - return getattr(model_info, "litellm_routing_preferences", None) + if isinstance(deployment_model_info, dict): + return deployment_model_info.get("litellm_routing_preferences") + return getattr(deployment_model_info, "litellm_routing_preferences", None) def _get_deployment_input_cost(self, deployment: object) -> float | None: """ @@ -323,7 +323,7 @@ class QualityRouter(CustomLogger): self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> Optional["PreRoutingHookResponse"]: diff --git a/litellm/router_utils/batch_utils.py b/litellm/router_utils/batch_utils.py index 5753d0d8db0..d71696a66b1 100644 --- a/litellm/router_utils/batch_utils.py +++ b/litellm/router_utils/batch_utils.py @@ -4,10 +4,14 @@ from collections.abc import Iterable from os import PathLike from typing import Final +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose from litellm.types.utils import CallTypes +_DECODED_JSONL_ROW: Final = TypeAdapter(object) + class InMemoryFile(io.BytesIO): def __init__(self, content: bytes, name: str, content_type: str = "application/jsonl"): @@ -42,7 +46,7 @@ def parse_jsonl_with_embedded_newlines(content: str) -> list[dict]: if char == "\n": # Try to parse what we have so far try: - json_object = json.loads(buffer.strip()) + json_object = _DECODED_JSONL_ROW.validate_python(json.loads(buffer.strip())) json_objects.append(json_object) buffer = "" # Reset buffer for next JSON object except json.JSONDecodeError: @@ -52,7 +56,7 @@ def parse_jsonl_with_embedded_newlines(content: str) -> list[dict]: # Handle any remaining content in buffer if buffer.strip(): try: - json_object = json.loads(buffer.strip()) + json_object = _DECODED_JSONL_ROW.validate_python(json.loads(buffer.strip())) json_objects.append(json_object) except json.JSONDecodeError as e: verbose_logger.error("error parsing final buffer: %s..., error: %s", buffer[:100], e) @@ -125,7 +129,7 @@ def replace_model_in_jsonl(file_content: FileTypes, new_model_name: str, keep_li buffer = "" continue try: - json_object = json.loads(stripped) + json_object = _DECODED_JSONL_ROW.validate_python(json.loads(stripped)) except json.JSONDecodeError: continue # object not complete yet; keep accumulating if isinstance(json_object, dict) and isinstance(json_object.get("body"), dict): diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 97bf4c590c2..e12304c6ed1 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -47,7 +47,8 @@ def request_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object: def is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool: if request_kwargs is None: return False - return getattr(request_user_api_key_auth(request_kwargs), "user_role", None) == "proxy_admin" + user_role: Final[object] = getattr(request_user_api_key_auth(request_kwargs), "user_role", None) + return user_role == "proxy_admin" _is_proxy_admin_request = is_proxy_admin_request diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 70b32f59f81..c7e3dab0d1a 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -47,7 +47,7 @@ if TYPE_CHECKING: from litellm.router import Router as _Router LitellmRouter = _Router - Span = _Span | Any + Span = _Span else: LitellmRouter = Any Span = Any diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index af637f1bdad..14ed6ed2adc 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from litellm.router import Router as _Router LitellmRouter = _Router - Span = _Span | Any + Span = _Span else: LitellmRouter = Any Span = Any diff --git a/litellm/router_utils/health_state_cache.py b/litellm/router_utils/health_state_cache.py index 4fb9476dae0..587c8015b79 100644 --- a/litellm/router_utils/health_state_cache.py +++ b/litellm/router_utils/health_state_cache.py @@ -20,7 +20,7 @@ HEALTH_CHECKS_TARGET: Final = "health_checks" if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 432fe11dc2e..88fb8876175 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -14,7 +14,7 @@ where routing to a consistent deployment is still beneficial. import hashlib from collections.abc import Callable, Mapping, Sequence -from typing import Any, Final, cast +from typing import Final, cast from typing_extensions import ReadOnly, TypedDict @@ -482,7 +482,9 @@ class DeploymentAffinityCheck(CustomLogger): ) return [deployment] - async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None: + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict | None: """ Persist/update the API-key -> deployment mapping for this request. diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index 17e7ec176ab..ea3fdd601dc 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -40,7 +40,7 @@ from litellm.utils import get_utc_datetime if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/sandbox/main.py b/litellm/sandbox/main.py index 04c2cef73ea..3d8e3e844f0 100644 --- a/litellm/sandbox/main.py +++ b/litellm/sandbox/main.py @@ -13,7 +13,7 @@ way `litellm.asearch` is. from typing import Final -import litellm +from litellm._logging import verbose_logger from litellm.llms.base_llm.sandbox.transformation import ( BaseSandboxConfig, CodeExecutionResult, @@ -143,4 +143,4 @@ async def acode_interpreter_tool( try: await config.adelete_sandbox(container=container, api_key=api_key, api_base=api_base, **forwarded) except Exception as e: - litellm._logging.verbose_logger.debug("sandbox: failed to delete ephemeral container: %s", e) + verbose_logger.debug("sandbox: failed to delete ephemeral container: %s", e) diff --git a/litellm/search/main.py b/litellm/search/main.py index fb19db5a483..96739b96e40 100644 --- a/litellm/search/main.py +++ b/litellm/search/main.py @@ -7,7 +7,7 @@ import contextvars from collections.abc import Coroutine, Mapping from functools import partial from types import MappingProxyType -from typing import Any, Final +from typing import Final import httpx from pydantic import TypeAdapter @@ -78,7 +78,7 @@ async def asearch( api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, object] | None = None, **kwargs, ) -> SearchResponse: """ @@ -181,9 +181,9 @@ def search( api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, - extra_headers: dict[str, Any] | None = None, + extra_headers: dict[str, object] | None = None, **kwargs, -) -> SearchResponse | Coroutine[Any, Any, SearchResponse]: +) -> SearchResponse | Coroutine[object, object, SearchResponse]: """ Synchronous Search function. diff --git a/litellm/secret_managers/aws_secret_manager.py b/litellm/secret_managers/aws_secret_manager.py index b95557789be..88642ae141e 100644 --- a/litellm/secret_managers/aws_secret_manager.py +++ b/litellm/secret_managers/aws_secret_manager.py @@ -88,7 +88,7 @@ class AWSKeyManagementService_V2: except Exception as e: raise e - def decrypt_value(self, secret_name: str) -> Any: + def decrypt_value(self, secret_name: str) -> str | bool: if self.kms_client is None: raise ValueError("kms_client is None") encrypted_value = os.getenv(secret_name, None) diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index 5d056ea3fe0..2b707f179d0 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -1,12 +1,15 @@ import os from collections.abc import Callable -from typing import Any, Final +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger from litellm.types.secret_managers.get_azure_ad_token_provider import ( AzureCredentialType, ) +if TYPE_CHECKING: + from azure.core.credentials import TokenProvider + def infer_credential_type_from_environment() -> AzureCredentialType: if ( @@ -74,9 +77,7 @@ def get_azure_ad_token_provider( else None or os.environ.get("AZURE_CREDENTIAL") or infer_credential_type_from_environment() ) verbose_logger.info("For Azure AD Token Provider, choosing credential type: %s", cred) - credential: ( - ClientSecretCredential | ManagedIdentityCredential | CertificateCredential | DefaultAzureCredential | Any | None - ) = None + credential: TokenProvider | None = None if cred == AzureCredentialType.ClientSecretCredential: credential = ClientSecretCredential( client_id=os.environ["AZURE_CLIENT_ID"], diff --git a/litellm/skills/main.py b/litellm/skills/main.py index 71fd78f11a3..6f57bd1bc96 100644 --- a/litellm/skills/main.py +++ b/litellm/skills/main.py @@ -267,7 +267,7 @@ async def alist_skills( page: str | None = None, source: str | None = None, extra_headers: dict[str, str] | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -330,7 +330,7 @@ def list_skills( page: str | None = None, source: str | None = None, extra_headers: dict[str, str] | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: Mapping[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py index ae40162c17c..ae2df8069a9 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py @@ -1,6 +1,7 @@ # Type definitions for DynamoAI Guardrails API import enum +from collections.abc import Mapping from typing import Any, Literal, TypedDict from pydantic import Field @@ -93,7 +94,7 @@ class DynamoAIProcessedResult(TypedDict): """Processed result from DynamoAI guardrail check""" violations_detected: list[str] - violation_details: dict[str, Any] + violation_details: Mapping[str, Mapping[str, object]] class DynamoAIGuardrailConfigModel(GuardrailConfigModel): diff --git a/litellm/utils.py b/litellm/utils.py index 1bc3b55992c..c0c80f09e4a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -24,7 +24,6 @@ import subprocess # What is this? ## Generic utils.py file. Problem-specific utils (e.g. 'cost calculation), should all be in `litellm_core_utils/`. -import sys import textwrap import threading import time @@ -282,10 +281,10 @@ try: .joinpath("anthropic_tokenizer.json") .open("r", encoding="utf-8") as f ): - json_data = json.load(f) + json_data = TypeAdapter(object).validate_python(json.load(f)) except (ImportError, AttributeError, TypeError): with resources.open_text("litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json") as f: - json_data = json.load(f) + json_data = TypeAdapter(object).validate_python(json.load(f)) # Convert to str (if necessary) claude_json_str = json.dumps(json_data) @@ -448,6 +447,8 @@ if TYPE_CHECKING: from litellm.types.rerank import RerankResponse from litellm.types.router import AllowedModelRegion, LiteLLM_Params + executor: BoundedLoggingThreadPoolExecutor + from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig @@ -1928,7 +1929,7 @@ def client(original_function): # Copy the current context to propagate it to the background thread # This is essential for OpenTelemetry span context propagation ctx: Final = contextvars.copy_context() - executor: Final[BoundedLoggingThreadPoolExecutor] = getattr(sys.modules[__name__], "executor") + executor: Final[BoundedLoggingThreadPoolExecutor] = litellm_utils.executor executor.submit( ctx.run, logging_obj.success_handler, @@ -3612,7 +3613,7 @@ def get_optional_params_transcription( passed_params: Final = locals() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") - custom_llm_provider = passed_params.pop("custom_llm_provider") + passed_params.pop("custom_llm_provider") passed_params.pop("drop_params") drop_params = normalize_drop_params(drop_params) special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") @@ -3716,7 +3717,7 @@ def get_optional_params_image_gen( provider_config: BaseImageGenerationConfig | None = None, drop_params: bool | None = None, **kwargs: object, -): +) -> dict[str, object]: # retrieve all parameters passed to the function passed_params: Final = locals() passed_params.pop("model", None) @@ -5187,10 +5188,10 @@ def add_provider_specific_params_to_optional_params( dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset( param for param in (additional_drop_params or ()) if isinstance(param, str) ) - processed_extra_body: Final = {k: v for k, v in initial_extra_body.items() if k not in dropped_keys} - - ensure_extra_body_is_safe: Final = getattr(sys.modules[__name__], "ensure_extra_body_is_safe") - optional_params["extra_body"] = ensure_extra_body_is_safe(extra_body=processed_extra_body) + ensure_extra_body_is_safe: Final = litellm_utils.ensure_extra_body_is_safe + optional_params["extra_body"] = ensure_extra_body_is_safe( + extra_body={k: v for k, v in initial_extra_body.items() if k not in dropped_keys} + ) else: for k in passed_params: if k not in openai_params and passed_params[k] is not None: @@ -7336,7 +7337,7 @@ _should_retry = should_retry def _get_retry_after_from_exception_header( - response_headers: httpx.Headers | None = None, + response_headers: Mapping[str, str] | None = None, ): """ Reimplementation of openai's calculate retry after, since that one can't be imported. @@ -7350,7 +7351,7 @@ def _get_retry_after_from_exception_header( # ". See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After#syntax for # details. if response_headers is not None: - retry_header: Final[str] = response_headers.get("retry-after") + retry_header: Final = response_headers.get("retry-after", "") try: retry_after = int(retry_header) except Exception: @@ -8109,16 +8110,16 @@ def get_base_model_from_metadata(model_call_details: Mapping[str, object] | None if base_model is not None: return cast(str | None, base_model) metadata: Final = cast(Mapping[str, object], litellm_params.get("metadata") or {}) - base_model_metadata_getter: Final[_BaseModelFromMetadataGetter] = getattr( - sys.modules[__name__], "get_base_model_from_litellm_call_metadata" + base_model_metadata_getter: Final[_BaseModelFromMetadataGetter] = ( + litellm_utils.get_base_model_from_litellm_call_metadata ) base_model_from_metadata: Final = base_model_metadata_getter(metadata=metadata) if base_model_from_metadata is not None: return base_model_from_metadata litellm_metadata: Final = cast(Mapping[str, object], litellm_params.get("litellm_metadata", {})) - litellm_metadata_getter: Final[_BaseModelFromMetadataGetter] = getattr( - sys.modules[__name__], "get_base_model_from_litellm_call_metadata" + litellm_metadata_getter: Final[_BaseModelFromMetadataGetter] = ( + litellm_utils.get_base_model_from_litellm_call_metadata ) return litellm_metadata_getter(metadata=litellm_metadata) return None diff --git a/litellm/vector_store_files/main.py b/litellm/vector_store_files/main.py index 3b6f1de3c7a..c03963980de 100644 --- a/litellm/vector_store_files/main.py +++ b/litellm/vector_store_files/main.py @@ -4,7 +4,7 @@ import asyncio import contextvars from collections.abc import Coroutine, Mapping from functools import partial -from typing import Any, Final +from typing import Final import httpx @@ -60,7 +60,7 @@ async def acreate( chunking_strategy: Mapping[str, object] | None = None, extra_headers: dict[str, str] | None = None, extra_query: Mapping[str, object] | None = None, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -112,7 +112,7 @@ def create( chunking_strategy: Mapping[str, object] | None = None, extra_headers: dict[str, str] | None = None, extra_query: Mapping[str, object] | None = None, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -188,7 +188,7 @@ async def alist( limit: int | None = None, order: str | None = None, extra_headers: dict[str, str] | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -241,7 +241,7 @@ def list( limit: int | None = None, order: str | None = None, extra_headers: dict[str, str] | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -527,7 +527,7 @@ async def aupdate( file_id: str, attributes: VectorStoreFileAttributes, extra_headers: dict[str, str] | None = None, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, @@ -573,7 +573,7 @@ def update( file_id: str, attributes: VectorStoreFileAttributes, extra_headers: dict[str, str] | None = None, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, diff --git a/litellm/vector_stores/utils.py b/litellm/vector_stores/utils.py index c0ac4edbf74..a2b5a6732a8 100644 --- a/litellm/vector_stores/utils.py +++ b/litellm/vector_stores/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Final, cast, get_type_hints +from typing import Final, cast, get_type_hints from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.types.vector_stores import ( @@ -12,7 +12,7 @@ class VectorStoreRequestUtils: @staticmethod def get_requested_vector_store_search_optional_param( - params: dict[str, Any], + params: dict[str, object], vector_store_provider_config: BaseVectorStoreConfig, ) -> VectorStoreSearchOptionalRequestParams: """ @@ -37,7 +37,7 @@ class VectorStoreRequestUtils: @staticmethod def get_requested_vector_store_create_optional_param( - params: dict[str, Any], + params: dict[str, object], ) -> VectorStoreCreateOptionalRequestParams: """ Filter parameters to only include those defined in VectorStoreCreateOptionalRequestParams.