diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 3180cea2568..da0f23372bf 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29682 + "limit": 29616 }, "reportArgumentType": { "limit": 2645 @@ -9,7 +9,7 @@ "limit": 329 }, "reportAttributeAccessIssue": { - "limit": 516 + "limit": 502 }, "reportCallIssue": { "limit": 123 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 9440 + "limit": 9366 }, "reportFunctionMemberAccess": { "limit": 11 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5848 + "limit": 5836 }, "reportMissingTypeArgument": { - "limit": 15850 + "limit": 15848 }, "reportMissingTypeStubs": { "limit": 41 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45297 + "limit": 45179 }, "reportUnknownLambdaType": { "limit": 113 }, "reportUnknownMemberType": { - "limit": 40411 + "limit": 40397 }, "reportUnknownParameterType": { - "limit": 20301 + "limit": 20285 }, "reportUnknownVariableType": { - "limit": 31968 + "limit": 31950 }, "reportUnnecessaryCast": { "limit": 177 diff --git a/litellm/main.py b/litellm/main.py index 8b2c3c72f76..6609280aea7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -19,7 +19,7 @@ import random import sys import time import traceback -from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping +from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping, Sequence from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy @@ -29,6 +29,7 @@ from typing import ( Any, Literal, Optional, + Protocol, Union, cast, get_args, @@ -40,6 +41,8 @@ from litellm._uuid import uuid if TYPE_CHECKING: from aiohttp import ClientSession + from litellm.types.llms.openai import AllMessageValues + import dotenv import httpx import openai @@ -334,6 +337,16 @@ MOCK_RESPONSE_TYPE = Union[str, Exception, dict, ModelResponse, ModelResponseStr ####### COMPLETION ENDPOINTS ################ +class _CompletionRouter(Protocol): + def completion( + self, model: str, messages: "Sequence[AllMessageValues]", **kwargs: object + ) -> ModelResponse | CustomStreamWrapper: ... + + async def acompletion( + self, model: str, messages: "Sequence[AllMessageValues]", **kwargs: object + ) -> ModelResponse | CustomStreamWrapper: ... + + class LiteLLM: def __init__( self, @@ -350,7 +363,7 @@ class LiteLLM: class Chat: - def __init__(self, params, router_obj: Any | None): + def __init__(self, params, router_obj: _CompletionRouter | None): self.params = params if self.params.get("acompletion", False) is True: self.params.pop("acompletion") @@ -360,11 +373,11 @@ class Chat: class Completions: - def __init__(self, params, router_obj: Any | None): + def __init__(self, params, router_obj: _CompletionRouter | None): self.params = params self.router_obj = router_obj - def create(self, messages, model=None, **kwargs): + def create(self, messages: "Sequence[AllMessageValues]", model: str | None = None, **kwargs: object): for k, v in kwargs.items(): self.params[k] = v model = model or self.params.get("model") @@ -376,11 +389,11 @@ class Completions: class AsyncCompletions: - def __init__(self, params, router_obj: Any | None): + def __init__(self, params, router_obj: _CompletionRouter | None): self.params = params self.router_obj = router_obj - async def create(self, messages, model=None, **kwargs): + async def create(self, messages: "Sequence[AllMessageValues]", model: str | None = None, **kwargs: object): for k, v in kwargs.items(): self.params[k] = v model = model or self.params.get("model") @@ -974,11 +987,11 @@ def responses_api_bridge_check( model: str, custom_llm_provider: str, web_search_options: OpenAIWebSearchOptions | None = None, - tools: list[Any] | None = None, - reasoning_effort: Any | None = None, - reasoning_summary: Any | None = None, -) -> tuple[dict, str]: - model_info: dict[str, Any] = {} + tools: Sequence[object] | None = None, + reasoning_effort: object | None = None, + reasoning_summary: object | None = None, +) -> tuple[Mapping[str, object], str]: + model_info: dict[str, object] = {} # mutable-ok: built incrementally across several branches below # Global flag: route ALL OpenAI chat completions through Responses API. # Returns early with minimal model_info; callers only inspect the "mode" key. @@ -1195,8 +1208,8 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): ## LOAD CONFIG - if set - config = litellm.AzureOpenAIO1Config.get_config() - for k, v in config.items(): + o1_config: Mapping[str, object] = litellm.AzureOpenAIO1Config.get_config() + for k, v in o1_config.items(): if ( k not in optional_params ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in @@ -1224,8 +1237,8 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul ) else: ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in config.items(): + azure_config: Mapping[str, object] = litellm.AzureOpenAIConfig.get_config() + for k, v in azure_config.items(): if ( k not in optional_params ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in @@ -1318,7 +1331,7 @@ def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatch optional_params["extra_headers"] = extra_headers ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() + config: Mapping[str, object] = litellm.AzureOpenAIConfig.get_config() for k, v in config.items(): if ( k not in optional_params @@ -1605,7 +1618,7 @@ def _complete_text_completion_openai( headers = headers or litellm.headers ## LOAD CONFIG - if set - config = litellm.OpenAITextCompletionConfig.get_config() + config: Mapping[str, object] = litellm.OpenAITextCompletionConfig.get_config() for k, v in config.items(): if ( k not in optional_params @@ -1889,7 +1902,7 @@ def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult headers = headers or litellm.headers ## LOAD CONFIG - if set - config = litellm.GroqChatConfig.get_config() + config: Mapping[str, object] = litellm.GroqChatConfig.get_config() for k, v in config.items(): if ( k not in optional_params @@ -1938,7 +1951,7 @@ def _complete_bedrock_mantle( api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") headers = headers or litellm.headers - config = litellm.BedrockMantleChatConfig.get_config() + config: Mapping[str, object] = litellm.BedrockMantleChatConfig.get_config() for k, v in config.items(): if k not in optional_params: optional_params[k] = v @@ -2106,7 +2119,7 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: headers = headers or litellm.headers ## LOAD CONFIG - if set - config = litellm.GenAIHubOrchestrationConfig.get_config() + config: Mapping[str, object] = litellm.GenAIHubOrchestrationConfig.get_config() for k, v in config.items(): if ( k not in optional_params @@ -2401,7 +2414,7 @@ def _complete_custom_openai( optional_params["metadata"] = openai_metadata ## LOAD CONFIG - if set - config = litellm.OpenAIConfig.get_config() + config: Mapping[str, object] = litellm.OpenAIConfig.get_config() for k, v in config.items(): if ( k not in optional_params @@ -3229,7 +3242,7 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch headers = openrouter_headers ## Load Config - config = litellm.OpenrouterConfig.get_config() + config: Mapping[str, object] = litellm.OpenrouterConfig.get_config() for k, v in config.items(): if k == "extra_body": # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models @@ -3307,7 +3320,7 @@ def _complete_vercel_ai_gateway( headers = vercel_headers ## Load Config - config = litellm.VercelAIGatewayConfig.get_config() + config: Mapping[str, object] = litellm.VercelAIGatewayConfig.get_config() for k, v in config.items(): if k == "extra_body": # we use openai 'extra_body' to pass vercel specific params - providerOptions @@ -4995,6 +5008,7 @@ def completion( # type: ignore # Inject proxy auth headers if configured if litellm.proxy_auth is not None: try: + # any-ok: importing ProxyAuthHandler would rebind litellm.proxy_auth via package auto-import proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: @@ -5103,12 +5117,15 @@ def completion( # type: ignore fallbacks = fallbacks or litellm.model_fallbacks if fallbacks is not None: return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime + # any-ok: args is completion()'s locals(); typeshed types locals() as dict[str, Any] **args ) if model_list is not None: deployments = [m["litellm_params"] for m in model_list if m["model_name"] == model] return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type - deployments=deployments, **args + deployments=deployments, + # any-ok: args is completion()'s locals(); typeshed types locals() as dict[str, Any] + **args, ) if litellm.model_alias_map and model in litellm.model_alias_map: model = litellm.model_alias_map[ @@ -5868,7 +5885,7 @@ def embedding( *, aembedding: Literal[True], **kwargs, -) -> Coroutine[Any, Any, EmbeddingResponse]: +) -> Coroutine[None, None, EmbeddingResponse]: ... @@ -5919,7 +5936,7 @@ def embedding( litellm_call_id=None, logger_fn=None, **kwargs, -) -> EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse]: +) -> EmbeddingResponse | Coroutine[None, None, EmbeddingResponse]: """ Embedding function that calls an API to generate embeddings for the given input. @@ -5962,6 +5979,7 @@ def embedding( # Inject proxy auth headers if configured if litellm.proxy_auth is not None: try: + # any-ok: importing ProxyAuthHandler would rebind litellm.proxy_auth via package auto-import proxy_headers = litellm.proxy_auth.get_auth_headers() headers.update(proxy_headers) except Exception as e: @@ -6039,7 +6057,7 @@ def embedding( if mock_response is not None: return mock_embedding(model=model, mock_response=mock_response) try: - response: EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse] | None = None + response: EmbeddingResponse | Coroutine[None, None, EmbeddingResponse] | None = None if azure is True or custom_llm_provider == "azure": # azure configs @@ -7289,7 +7307,7 @@ async def aadapter_generate_content( from litellm.google_genai.adapters.handler import GenerateContentToCompletionHandler coro = cast( - Coroutine[Any, Any, dict[str, Any] | AsyncIterator[bytes]], + Coroutine[None, None, dict[str, Any] | AsyncIterator[bytes]], GenerateContentToCompletionHandler.generate_content_handler(**kwargs, _is_async=True), ) return await coro @@ -7496,7 +7514,7 @@ def transcription( max_retries: int | None = None, custom_llm_provider=None, **kwargs, -) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: +) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: """ Calls openai + azure whisper endpoints. @@ -7563,7 +7581,7 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse] | None = None + response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None provider_config = ProviderConfigManager.get_provider_audio_transcription_config( model=model, @@ -7797,7 +7815,7 @@ def speech( custom_llm_provider: str | None = None, aspeech: bool | None = None, **kwargs, -) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: +) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: user = kwargs.get("user", None) litellm_call_id: str | None = kwargs.get("litellm_call_id", None) proxy_server_request = kwargs.get("proxy_server_request", None) @@ -7856,7 +7874,7 @@ def speech( }, custom_llm_provider=custom_llm_provider, ) - response: HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent] | None = None + response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers: if voice is None or not (isinstance(voice, str)): raise litellm.BadRequestError( @@ -8780,17 +8798,18 @@ async def acount_tokens( # Cache for encoding to avoid repeated __getattr__ calls -_encoding_cache: Any | None = None +_encoding_cache: tiktoken.Encoding | None = None -def _get_encoding(): +def _get_encoding() -> tiktoken.Encoding: """Get encoding, loading it lazily if needed.""" global _encoding_cache if _encoding_cache is None: import sys # Access via module to trigger __getattr__ if not cached - _encoding_cache = sys.modules[__name__].encoding + _encoding_cache = sys.modules[__name__].encoding # any-ok: __getattr__ module hook; attribute type is dynamic + assert _encoding_cache is not None return _encoding_cache @@ -8802,7 +8821,8 @@ def __getattr__(name: str) -> Any: # instead of downloading from the internet from litellm._lazy_imports import _get_default_encoding - _encoding = _get_default_encoding() + # any-ok: _get_default_encoding() is declared -> Any in _lazy_imports.py + _encoding: tiktoken.Encoding = _get_default_encoding() # Cache it in the module's __dict__ for subsequent accesses import sys diff --git a/litellm/router.py b/litellm/router.py index 7d6499cf7d2..60f4a7c2815 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -249,7 +249,7 @@ if TYPE_CHECKING: ResponsesAPIResponse, ) - Span = Union[_Span, Any] + Span = _Span else: Span = Any AutoRouter = Any @@ -307,6 +307,14 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") +RoutingStrategySelector = ( + LeastBusyLoggingHandler + | LowestTPMLoggingHandler + | LowestTPMLoggingHandler_v2 + | LowestLatencyLoggingHandler + | LowestCostLoggingHandler +) + class RoutingArgs(enum.Enum): ttl = 60 # 1min (RPM/TPM expire key) @@ -669,6 +677,9 @@ class Router: """ ### ROUTING SETUP ### + self._override_selectors: dict[str, RoutingStrategySelector | None] = {} + self._override_selectors_lock = threading.Lock() + self._group_selectors: dict[str, dict[str, RoutingStrategySelector]] = {} if self._normalize_strategy(routing_strategy) == "lar1": from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy @@ -679,8 +690,6 @@ class Router: routing_strategy_args=routing_strategy_args, ) self._init_routing_groups(self._routing_groups_input) - self._override_selectors: dict[str, Any] = {} - self._override_selectors_lock = threading.Lock() self.access_groups = None ## USAGE TRACKING ## if isinstance(litellm._async_success_callback, list): @@ -884,13 +893,13 @@ class Router: strategy: RoutingStrategy | str, routing_strategy_args: dict, register_callbacks: bool = True, - ) -> Any | None: + ) -> RoutingStrategySelector | None: """ Constructs a strategy selector for a given strategy. Returns None for `simple-shuffle` (no selector needed) and unknown strategies. """ - selector: Any | None = None + selector: RoutingStrategySelector | None = None match self._normalize_strategy(strategy): case RoutingStrategy.LEAST_BUSY.value: selector = LeastBusyLoggingHandler(router_cache=self.cache) @@ -927,7 +936,7 @@ class Router: return selector - def _unregister_router_selectors(self, selectors: list[Any]) -> None: + def _unregister_router_selectors(self, selectors: list[RoutingStrategySelector | None]) -> None: """ Drop router-owned strategy selectors from litellm's global callback lists by identity. Used before re-init (`routing_strategy_init` / @@ -949,7 +958,7 @@ class Router: self._unregister_router_selectors( [getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()] - + list(getattr(self, "_override_selectors", {}).values()) + + list(self._override_selectors.values()) ) self._override_selectors = {} @@ -985,12 +994,12 @@ class Router: attributes set up in `routing_strategy_init`. """ self._unregister_router_selectors( - [sel for selectors in getattr(self, "_group_selectors", {}).values() for sel in selectors.values()] + [sel for selectors in self._group_selectors.values() for sel in selectors.values()] ) self._routing_groups: dict[str, RoutingGroup] = {} self._model_to_group: dict[str, str] = {} - self._group_selectors: dict[str, dict[str, Any]] = {} + self._group_selectors = {} if not groups_input: return @@ -1068,7 +1077,7 @@ class Router: return None return strategy - def _get_override_strategy_selector(self, strategy: str) -> Any | None: + def _get_override_strategy_selector(self, strategy: str) -> RoutingStrategySelector | None: """ Returns the selector for a per-request strategy override. @@ -1089,7 +1098,9 @@ class Router: ) return self._override_selectors[strategy] - def _get_routing_context(self, model: str, request_kwargs: dict | None = None) -> tuple[str | None, Any | None]: + def _get_routing_context( + self, model: str, request_kwargs: dict | None = None + ) -> tuple[str | None, RoutingStrategySelector | None]: """ Resolves the routing strategy and selector to use for the given model. @@ -1575,9 +1586,11 @@ class Router: EncryptedContentAffinityCheck, ) + _CallbackT = TypeVar("_CallbackT", bound=CustomLogger | Callable[..., object] | str) + def _move_before_deployment_affinity( - callback_list: list[Any], - callback_to_move: EncryptedContentAffinityCheck, + callback_list: list[_CallbackT], + callback_to_move: _CallbackT, ) -> None: if callback_to_move not in callback_list: return @@ -1883,7 +1896,7 @@ class Router: return silent_kwargs - def _silent_experiment_completion(self, silent_model: str, messages: list[Any], **kwargs): + def _silent_experiment_completion(self, silent_model: str, messages: list[AllMessageValues], **kwargs): """ Run a silent experiment in the background (thread). """ @@ -1910,7 +1923,7 @@ class Router: async def _run_silent_completion(): await self.acompletion( model=silent_model, - messages=cast(list[AllMessageValues], messages), + messages=messages, **silent_kwargs, ) # Drain any fire-and-forget tasks (e.g. alerting hooks) @@ -5632,7 +5645,7 @@ class Router: def sync_wrapper( custom_llm_provider: str | None = None, - client: Any | None = None, + client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.) **kwargs, ): return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) @@ -5648,7 +5661,7 @@ class Router: def vector_store_sync_wrapper( custom_llm_provider: str | None = None, - client: Any | None = None, + client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.) **kwargs, ): if custom_llm_provider and "custom_llm_provider" not in kwargs: @@ -5670,7 +5683,7 @@ class Router: def vector_store_file_sync_wrapper( custom_llm_provider: str | None = None, - client: Any | None = None, + client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.) **kwargs, ): return original_function( @@ -5691,7 +5704,7 @@ class Router: def managed_agents_sync_wrapper( custom_llm_provider: str | None = None, - client: Any | None = None, + client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.) **kwargs, ): if custom_llm_provider and "custom_llm_provider" not in kwargs: @@ -5705,7 +5718,7 @@ class Router: # Handle asynchronous call types async def async_wrapper( custom_llm_provider: str | None = None, - client: Any | None = None, + client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.) **kwargs, ): if call_type == "assistants": @@ -7079,7 +7092,9 @@ class Router: except Exception as e: raise e - async def async_deployment_callback_on_failure(self, kwargs, completion_response: Any | None, start_time, end_time): + async def async_deployment_callback_on_failure( + self, kwargs, completion_response: object | None, start_time, end_time + ): """ Update RPM usage for a deployment """ @@ -8650,7 +8665,7 @@ class Router: def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None - ) -> dict[str, Any] | None: + ) -> dict[str, Any] | None: # any-ok: pydantic BaseModel.model_dump() is typed to return dict[str, Any] """ Get API credentials and provider info from a model name in model_list. Useful for passthrough endpoints (files, batches, etc.) that need credentials. @@ -10999,7 +11014,7 @@ class Router: self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None, + messages: list[dict[str, str]] | None, ) -> RoutingContext: """ Build a RoutingContext for `model`, run it through `self.routing_plugins` @@ -11098,7 +11113,7 @@ class Router: self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, str]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, ) -> PreRoutingHookResponse | None: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index d27b168d6ca..919d6f86cce 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,18 +1,18 @@ { "ANN001": { - "limit": 3097 + "limit": 3089 }, "ANN002": { "limit": 69 }, "ANN003": { - "limit": 831 + "limit": 827 }, "ANN201": { "limit": 2137 }, "ANN202": { - "limit": 941 + "limit": 939 }, "ANN204": { "limit": 724 @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 1848 + "limit": 1832 }, "ASYNC230": { "limit": 14 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 289c0a0afd6..30f04efe730 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23349 + "limit": 23346 }, "LIT002": { - "limit": 27252 + "limit": 27249 }, "LIT003": { "limit": 292 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1105 + "limit": 1103 }, "LIT007": { "limit": 0