mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
chore(typing): clear basedpyright Any errors in main.py and router.py
Replace Any-typed seams in litellm/main.py and litellm/router.py with concrete types (Protocols, Mapping/Sequence views, TypeVars, real provider types) so basedpyright can verify more of both files. Pure typing pass, no behavior change.
This commit is contained in:
mateo-berri
parent
956d5177d1
commit
a14620c0dd
5 changed files with 111 additions and 76 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"reportAny": {
|
||||
"limit": 29682
|
||||
"limit": 29616
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2645
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
"limit": 329
|
||||
},
|
||||
"reportAttributeAccessIssue": {
|
||||
"limit": 516
|
||||
"limit": 502
|
||||
},
|
||||
"reportCallIssue": {
|
||||
"limit": 123
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 42
|
||||
},
|
||||
"reportExplicitAny": {
|
||||
"limit": 9440
|
||||
"limit": 9366
|
||||
},
|
||||
"reportFunctionMemberAccess": {
|
||||
"limit": 11
|
||||
|
|
@ -54,10 +54,10 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportMissingParameterType": {
|
||||
"limit": 5848
|
||||
"limit": 5836
|
||||
},
|
||||
"reportMissingTypeArgument": {
|
||||
"limit": 15850
|
||||
"limit": 15848
|
||||
},
|
||||
"reportMissingTypeStubs": {
|
||||
"limit": 41
|
||||
|
|
@ -99,19 +99,19 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 45297
|
||||
"limit": 45179
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 113
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 40411
|
||||
"limit": 40397
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 20301
|
||||
"limit": 20285
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 31968
|
||||
"limit": 31950
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 177
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import random
|
|||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterable, Mapping, Sequence
|
||||
from concurrent import futures
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from copy import deepcopy
|
||||
|
|
@ -29,6 +29,7 @@ from typing import (
|
|||
Any,
|
||||
Literal,
|
||||
Optional,
|
||||
Protocol,
|
||||
Union,
|
||||
cast,
|
||||
get_args,
|
||||
|
|
@ -40,6 +41,8 @@ from litellm._uuid import uuid
|
|||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
import dotenv
|
||||
import httpx
|
||||
import openai
|
||||
|
|
@ -334,6 +337,16 @@ MOCK_RESPONSE_TYPE = Union[str, Exception, dict, ModelResponse, ModelResponseStr
|
|||
####### COMPLETION ENDPOINTS ################
|
||||
|
||||
|
||||
class _CompletionRouter(Protocol):
|
||||
def completion(
|
||||
self, model: str, messages: "Sequence[AllMessageValues]", **kwargs: object
|
||||
) -> ModelResponse | CustomStreamWrapper: ...
|
||||
|
||||
async def acompletion(
|
||||
self, model: str, messages: "Sequence[AllMessageValues]", **kwargs: object
|
||||
) -> ModelResponse | CustomStreamWrapper: ...
|
||||
|
||||
|
||||
class LiteLLM:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -350,7 +363,7 @@ class LiteLLM:
|
|||
|
||||
|
||||
class Chat:
|
||||
def __init__(self, params, router_obj: Any | None):
|
||||
def __init__(self, params, router_obj: _CompletionRouter | None):
|
||||
self.params = params
|
||||
if self.params.get("acompletion", False) is True:
|
||||
self.params.pop("acompletion")
|
||||
|
|
@ -360,11 +373,11 @@ class Chat:
|
|||
|
||||
|
||||
class Completions:
|
||||
def __init__(self, params, router_obj: Any | None):
|
||||
def __init__(self, params, router_obj: _CompletionRouter | None):
|
||||
self.params = params
|
||||
self.router_obj = router_obj
|
||||
|
||||
def create(self, messages, model=None, **kwargs):
|
||||
def create(self, messages: "Sequence[AllMessageValues]", model: str | None = None, **kwargs: object):
|
||||
for k, v in kwargs.items():
|
||||
self.params[k] = v
|
||||
model = model or self.params.get("model")
|
||||
|
|
@ -376,11 +389,11 @@ class Completions:
|
|||
|
||||
|
||||
class AsyncCompletions:
|
||||
def __init__(self, params, router_obj: Any | None):
|
||||
def __init__(self, params, router_obj: _CompletionRouter | None):
|
||||
self.params = params
|
||||
self.router_obj = router_obj
|
||||
|
||||
async def create(self, messages, model=None, **kwargs):
|
||||
async def create(self, messages: "Sequence[AllMessageValues]", model: str | None = None, **kwargs: object):
|
||||
for k, v in kwargs.items():
|
||||
self.params[k] = v
|
||||
model = model or self.params.get("model")
|
||||
|
|
@ -974,11 +987,11 @@ def responses_api_bridge_check(
|
|||
model: str,
|
||||
custom_llm_provider: str,
|
||||
web_search_options: OpenAIWebSearchOptions | None = None,
|
||||
tools: list[Any] | None = None,
|
||||
reasoning_effort: Any | None = None,
|
||||
reasoning_summary: Any | None = None,
|
||||
) -> tuple[dict, str]:
|
||||
model_info: dict[str, Any] = {}
|
||||
tools: Sequence[object] | None = None,
|
||||
reasoning_effort: object | None = None,
|
||||
reasoning_summary: object | None = None,
|
||||
) -> tuple[Mapping[str, object], str]:
|
||||
model_info: dict[str, object] = {} # mutable-ok: built incrementally across several branches below
|
||||
|
||||
# Global flag: route ALL OpenAI chat completions through Responses API.
|
||||
# Returns early with minimal model_info; callers only inspect the "mode" key.
|
||||
|
|
@ -1195,8 +1208,8 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
|
|||
|
||||
if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model):
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIO1Config.get_config()
|
||||
for k, v in config.items():
|
||||
o1_config: Mapping[str, object] = litellm.AzureOpenAIO1Config.get_config()
|
||||
for k, v in o1_config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
|
|
@ -1224,8 +1237,8 @@ def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResul
|
|||
)
|
||||
else:
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in config.items():
|
||||
azure_config: Mapping[str, object] = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in azure_config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in
|
||||
|
|
@ -1318,7 +1331,7 @@ def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatch
|
|||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
@ -1605,7 +1618,7 @@ def _complete_text_completion_openai(
|
|||
headers = headers or litellm.headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.OpenAITextCompletionConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.OpenAITextCompletionConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
@ -1889,7 +1902,7 @@ def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult
|
|||
headers = headers or litellm.headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.GroqChatConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.GroqChatConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
@ -1938,7 +1951,7 @@ def _complete_bedrock_mantle(
|
|||
api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
|
||||
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
|
||||
headers = headers or litellm.headers
|
||||
config = litellm.BedrockMantleChatConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.BedrockMantleChatConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if k not in optional_params:
|
||||
optional_params[k] = v
|
||||
|
|
@ -2106,7 +2119,7 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
|
|||
|
||||
headers = headers or litellm.headers
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.GenAIHubOrchestrationConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.GenAIHubOrchestrationConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
@ -2401,7 +2414,7 @@ def _complete_custom_openai(
|
|||
optional_params["metadata"] = openai_metadata
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.OpenAIConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.OpenAIConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
@ -3229,7 +3242,7 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch
|
|||
headers = openrouter_headers
|
||||
|
||||
## Load Config
|
||||
config = litellm.OpenrouterConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.OpenrouterConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if k == "extra_body":
|
||||
# we use openai 'extra_body' to pass openrouter specific params - transforms, route, models
|
||||
|
|
@ -3307,7 +3320,7 @@ def _complete_vercel_ai_gateway(
|
|||
headers = vercel_headers
|
||||
|
||||
## Load Config
|
||||
config = litellm.VercelAIGatewayConfig.get_config()
|
||||
config: Mapping[str, object] = litellm.VercelAIGatewayConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if k == "extra_body":
|
||||
# we use openai 'extra_body' to pass vercel specific params - providerOptions
|
||||
|
|
@ -4995,6 +5008,7 @@ def completion( # type: ignore
|
|||
# Inject proxy auth headers if configured
|
||||
if litellm.proxy_auth is not None:
|
||||
try:
|
||||
# any-ok: importing ProxyAuthHandler would rebind litellm.proxy_auth via package auto-import
|
||||
proxy_headers = litellm.proxy_auth.get_auth_headers()
|
||||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
|
|
@ -5103,12 +5117,15 @@ def completion( # type: ignore
|
|||
fallbacks = fallbacks or litellm.model_fallbacks
|
||||
if fallbacks is not None:
|
||||
return completion_with_fallbacks( # pyright: ignore[reportReturnType] # fallback runner is untyped; resolves to ModelResponse|CustomStreamWrapper at runtime
|
||||
# any-ok: args is completion()'s locals(); typeshed types locals() as dict[str, Any]
|
||||
**args
|
||||
)
|
||||
if model_list is not None:
|
||||
deployments = [m["litellm_params"] for m in model_list if m["model_name"] == model]
|
||||
return litellm.batch_completion_models( # pyright: ignore[reportReturnType] # batch path returns a list of responses, outside completion()'s single-response return type
|
||||
deployments=deployments, **args
|
||||
deployments=deployments,
|
||||
# any-ok: args is completion()'s locals(); typeshed types locals() as dict[str, Any]
|
||||
**args,
|
||||
)
|
||||
if litellm.model_alias_map and model in litellm.model_alias_map:
|
||||
model = litellm.model_alias_map[
|
||||
|
|
@ -5868,7 +5885,7 @@ def embedding(
|
|||
*,
|
||||
aembedding: Literal[True],
|
||||
**kwargs,
|
||||
) -> Coroutine[Any, Any, EmbeddingResponse]:
|
||||
) -> Coroutine[None, None, EmbeddingResponse]:
|
||||
...
|
||||
|
||||
|
||||
|
|
@ -5919,7 +5936,7 @@ def embedding(
|
|||
litellm_call_id=None,
|
||||
logger_fn=None,
|
||||
**kwargs,
|
||||
) -> EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse]:
|
||||
) -> EmbeddingResponse | Coroutine[None, None, EmbeddingResponse]:
|
||||
"""
|
||||
Embedding function that calls an API to generate embeddings for the given input.
|
||||
|
||||
|
|
@ -5962,6 +5979,7 @@ def embedding(
|
|||
# Inject proxy auth headers if configured
|
||||
if litellm.proxy_auth is not None:
|
||||
try:
|
||||
# any-ok: importing ProxyAuthHandler would rebind litellm.proxy_auth via package auto-import
|
||||
proxy_headers = litellm.proxy_auth.get_auth_headers()
|
||||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
|
|
@ -6039,7 +6057,7 @@ def embedding(
|
|||
if mock_response is not None:
|
||||
return mock_embedding(model=model, mock_response=mock_response)
|
||||
try:
|
||||
response: EmbeddingResponse | Coroutine[Any, Any, EmbeddingResponse] | None = None
|
||||
response: EmbeddingResponse | Coroutine[None, None, EmbeddingResponse] | None = None
|
||||
|
||||
if azure is True or custom_llm_provider == "azure":
|
||||
# azure configs
|
||||
|
|
@ -7289,7 +7307,7 @@ async def aadapter_generate_content(
|
|||
from litellm.google_genai.adapters.handler import GenerateContentToCompletionHandler
|
||||
|
||||
coro = cast(
|
||||
Coroutine[Any, Any, dict[str, Any] | AsyncIterator[bytes]],
|
||||
Coroutine[None, None, dict[str, Any] | AsyncIterator[bytes]],
|
||||
GenerateContentToCompletionHandler.generate_content_handler(**kwargs, _is_async=True),
|
||||
)
|
||||
return await coro
|
||||
|
|
@ -7496,7 +7514,7 @@ def transcription(
|
|||
max_retries: int | None = None,
|
||||
custom_llm_provider=None,
|
||||
**kwargs,
|
||||
) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]:
|
||||
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
|
||||
"""
|
||||
Calls openai + azure whisper endpoints.
|
||||
|
||||
|
|
@ -7563,7 +7581,7 @@ def transcription(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
response: TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse] | None = None
|
||||
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None
|
||||
|
||||
provider_config = ProviderConfigManager.get_provider_audio_transcription_config(
|
||||
model=model,
|
||||
|
|
@ -7797,7 +7815,7 @@ def speech(
|
|||
custom_llm_provider: str | None = None,
|
||||
aspeech: bool | None = None,
|
||||
**kwargs,
|
||||
) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]:
|
||||
) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]:
|
||||
user = kwargs.get("user", None)
|
||||
litellm_call_id: str | None = kwargs.get("litellm_call_id", None)
|
||||
proxy_server_request = kwargs.get("proxy_server_request", None)
|
||||
|
|
@ -7856,7 +7874,7 @@ def speech(
|
|||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
response: HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent] | None = None
|
||||
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
|
||||
if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers:
|
||||
if voice is None or not (isinstance(voice, str)):
|
||||
raise litellm.BadRequestError(
|
||||
|
|
@ -8780,17 +8798,18 @@ async def acount_tokens(
|
|||
|
||||
|
||||
# Cache for encoding to avoid repeated __getattr__ calls
|
||||
_encoding_cache: Any | None = None
|
||||
_encoding_cache: tiktoken.Encoding | None = None
|
||||
|
||||
|
||||
def _get_encoding():
|
||||
def _get_encoding() -> tiktoken.Encoding:
|
||||
"""Get encoding, loading it lazily if needed."""
|
||||
global _encoding_cache
|
||||
if _encoding_cache is None:
|
||||
import sys
|
||||
|
||||
# Access via module to trigger __getattr__ if not cached
|
||||
_encoding_cache = sys.modules[__name__].encoding
|
||||
_encoding_cache = sys.modules[__name__].encoding # any-ok: __getattr__ module hook; attribute type is dynamic
|
||||
assert _encoding_cache is not None
|
||||
return _encoding_cache
|
||||
|
||||
|
||||
|
|
@ -8802,7 +8821,8 @@ def __getattr__(name: str) -> Any:
|
|||
# instead of downloading from the internet
|
||||
from litellm._lazy_imports import _get_default_encoding
|
||||
|
||||
_encoding = _get_default_encoding()
|
||||
# any-ok: _get_default_encoding() is declared -> Any in _lazy_imports.py
|
||||
_encoding: tiktoken.Encoding = _get_default_encoding()
|
||||
# Cache it in the module's __dict__ for subsequent accesses
|
||||
import sys
|
||||
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ if TYPE_CHECKING:
|
|||
ResponsesAPIResponse,
|
||||
)
|
||||
|
||||
Span = Union[_Span, Any]
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
AutoRouter = Any
|
||||
|
|
@ -307,6 +307,14 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
|
|||
|
||||
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
||||
|
||||
RoutingStrategySelector = (
|
||||
LeastBusyLoggingHandler
|
||||
| LowestTPMLoggingHandler
|
||||
| LowestTPMLoggingHandler_v2
|
||||
| LowestLatencyLoggingHandler
|
||||
| LowestCostLoggingHandler
|
||||
)
|
||||
|
||||
|
||||
class RoutingArgs(enum.Enum):
|
||||
ttl = 60 # 1min (RPM/TPM expire key)
|
||||
|
|
@ -669,6 +677,9 @@ class Router:
|
|||
"""
|
||||
|
||||
### ROUTING SETUP ###
|
||||
self._override_selectors: dict[str, RoutingStrategySelector | None] = {}
|
||||
self._override_selectors_lock = threading.Lock()
|
||||
self._group_selectors: dict[str, dict[str, RoutingStrategySelector]] = {}
|
||||
if self._normalize_strategy(routing_strategy) == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
|
|
@ -679,8 +690,6 @@ class Router:
|
|||
routing_strategy_args=routing_strategy_args,
|
||||
)
|
||||
self._init_routing_groups(self._routing_groups_input)
|
||||
self._override_selectors: dict[str, Any] = {}
|
||||
self._override_selectors_lock = threading.Lock()
|
||||
self.access_groups = None
|
||||
## USAGE TRACKING ##
|
||||
if isinstance(litellm._async_success_callback, list):
|
||||
|
|
@ -884,13 +893,13 @@ class Router:
|
|||
strategy: RoutingStrategy | str,
|
||||
routing_strategy_args: dict,
|
||||
register_callbacks: bool = True,
|
||||
) -> Any | None:
|
||||
) -> RoutingStrategySelector | None:
|
||||
"""
|
||||
Constructs a strategy selector for a given strategy.
|
||||
Returns None for `simple-shuffle` (no selector needed) and unknown
|
||||
strategies.
|
||||
"""
|
||||
selector: Any | None = None
|
||||
selector: RoutingStrategySelector | None = None
|
||||
match self._normalize_strategy(strategy):
|
||||
case RoutingStrategy.LEAST_BUSY.value:
|
||||
selector = LeastBusyLoggingHandler(router_cache=self.cache)
|
||||
|
|
@ -927,7 +936,7 @@ class Router:
|
|||
|
||||
return selector
|
||||
|
||||
def _unregister_router_selectors(self, selectors: list[Any]) -> None:
|
||||
def _unregister_router_selectors(self, selectors: list[RoutingStrategySelector | None]) -> None:
|
||||
"""
|
||||
Drop router-owned strategy selectors from litellm's global callback
|
||||
lists by identity. Used before re-init (`routing_strategy_init` /
|
||||
|
|
@ -949,7 +958,7 @@ class Router:
|
|||
|
||||
self._unregister_router_selectors(
|
||||
[getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()]
|
||||
+ list(getattr(self, "_override_selectors", {}).values())
|
||||
+ list(self._override_selectors.values())
|
||||
)
|
||||
self._override_selectors = {}
|
||||
|
||||
|
|
@ -985,12 +994,12 @@ class Router:
|
|||
attributes set up in `routing_strategy_init`.
|
||||
"""
|
||||
self._unregister_router_selectors(
|
||||
[sel for selectors in getattr(self, "_group_selectors", {}).values() for sel in selectors.values()]
|
||||
[sel for selectors in self._group_selectors.values() for sel in selectors.values()]
|
||||
)
|
||||
|
||||
self._routing_groups: dict[str, RoutingGroup] = {}
|
||||
self._model_to_group: dict[str, str] = {}
|
||||
self._group_selectors: dict[str, dict[str, Any]] = {}
|
||||
self._group_selectors = {}
|
||||
|
||||
if not groups_input:
|
||||
return
|
||||
|
|
@ -1068,7 +1077,7 @@ class Router:
|
|||
return None
|
||||
return strategy
|
||||
|
||||
def _get_override_strategy_selector(self, strategy: str) -> Any | None:
|
||||
def _get_override_strategy_selector(self, strategy: str) -> RoutingStrategySelector | None:
|
||||
"""
|
||||
Returns the selector for a per-request strategy override.
|
||||
|
||||
|
|
@ -1089,7 +1098,9 @@ class Router:
|
|||
)
|
||||
return self._override_selectors[strategy]
|
||||
|
||||
def _get_routing_context(self, model: str, request_kwargs: dict | None = None) -> tuple[str | None, Any | None]:
|
||||
def _get_routing_context(
|
||||
self, model: str, request_kwargs: dict | None = None
|
||||
) -> tuple[str | None, RoutingStrategySelector | None]:
|
||||
"""
|
||||
Resolves the routing strategy and selector to use for the given model.
|
||||
|
||||
|
|
@ -1575,9 +1586,11 @@ class Router:
|
|||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
_CallbackT = TypeVar("_CallbackT", bound=CustomLogger | Callable[..., object] | str)
|
||||
|
||||
def _move_before_deployment_affinity(
|
||||
callback_list: list[Any],
|
||||
callback_to_move: EncryptedContentAffinityCheck,
|
||||
callback_list: list[_CallbackT],
|
||||
callback_to_move: _CallbackT,
|
||||
) -> None:
|
||||
if callback_to_move not in callback_list:
|
||||
return
|
||||
|
|
@ -1883,7 +1896,7 @@ class Router:
|
|||
|
||||
return silent_kwargs
|
||||
|
||||
def _silent_experiment_completion(self, silent_model: str, messages: list[Any], **kwargs):
|
||||
def _silent_experiment_completion(self, silent_model: str, messages: list[AllMessageValues], **kwargs):
|
||||
"""
|
||||
Run a silent experiment in the background (thread).
|
||||
"""
|
||||
|
|
@ -1910,7 +1923,7 @@ class Router:
|
|||
async def _run_silent_completion():
|
||||
await self.acompletion(
|
||||
model=silent_model,
|
||||
messages=cast(list[AllMessageValues], messages),
|
||||
messages=messages,
|
||||
**silent_kwargs,
|
||||
)
|
||||
# Drain any fire-and-forget tasks (e.g. alerting hooks)
|
||||
|
|
@ -5632,7 +5645,7 @@ class Router:
|
|||
|
||||
def sync_wrapper(
|
||||
custom_llm_provider: str | None = None,
|
||||
client: Any | None = None,
|
||||
client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.)
|
||||
**kwargs,
|
||||
):
|
||||
return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs)
|
||||
|
|
@ -5648,7 +5661,7 @@ class Router:
|
|||
|
||||
def vector_store_sync_wrapper(
|
||||
custom_llm_provider: str | None = None,
|
||||
client: Any | None = None,
|
||||
client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.)
|
||||
**kwargs,
|
||||
):
|
||||
if custom_llm_provider and "custom_llm_provider" not in kwargs:
|
||||
|
|
@ -5670,7 +5683,7 @@ class Router:
|
|||
|
||||
def vector_store_file_sync_wrapper(
|
||||
custom_llm_provider: str | None = None,
|
||||
client: Any | None = None,
|
||||
client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.)
|
||||
**kwargs,
|
||||
):
|
||||
return original_function(
|
||||
|
|
@ -5691,7 +5704,7 @@ class Router:
|
|||
|
||||
def managed_agents_sync_wrapper(
|
||||
custom_llm_provider: str | None = None,
|
||||
client: Any | None = None,
|
||||
client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.)
|
||||
**kwargs,
|
||||
):
|
||||
if custom_llm_provider and "custom_llm_provider" not in kwargs:
|
||||
|
|
@ -5705,7 +5718,7 @@ class Router:
|
|||
# Handle asynchronous call types
|
||||
async def async_wrapper(
|
||||
custom_llm_provider: str | None = None,
|
||||
client: Any | None = None,
|
||||
client: Any | None = None, # any-ok: provider SDK client varies by call_type (AsyncOpenAI, etc.)
|
||||
**kwargs,
|
||||
):
|
||||
if call_type == "assistants":
|
||||
|
|
@ -7079,7 +7092,9 @@ class Router:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
async def async_deployment_callback_on_failure(self, kwargs, completion_response: Any | None, start_time, end_time):
|
||||
async def async_deployment_callback_on_failure(
|
||||
self, kwargs, completion_response: object | None, start_time, end_time
|
||||
):
|
||||
"""
|
||||
Update RPM usage for a deployment
|
||||
"""
|
||||
|
|
@ -8650,7 +8665,7 @@ class Router:
|
|||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
) -> dict[str, Any] | None:
|
||||
) -> dict[str, Any] | None: # any-ok: pydantic BaseModel.model_dump() is typed to return dict[str, Any]
|
||||
"""
|
||||
Get API credentials and provider info from a model name in model_list.
|
||||
Useful for passthrough endpoints (files, batches, etc.) that need credentials.
|
||||
|
|
@ -10999,7 +11014,7 @@ class Router:
|
|||
self,
|
||||
model: str,
|
||||
request_kwargs: dict,
|
||||
messages: list[dict[str, Any]] | None,
|
||||
messages: list[dict[str, str]] | None,
|
||||
) -> RoutingContext:
|
||||
"""
|
||||
Build a RoutingContext for `model`, run it through `self.routing_plugins`
|
||||
|
|
@ -11098,7 +11113,7 @@ class Router:
|
|||
self,
|
||||
model: str,
|
||||
request_kwargs: dict,
|
||||
messages: list[dict[str, Any]] | None = None,
|
||||
messages: list[dict[str, str]] | None = None,
|
||||
input: str | list | None = None,
|
||||
specific_deployment: bool | None = False,
|
||||
) -> PreRoutingHookResponse | None:
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
{
|
||||
"ANN001": {
|
||||
"limit": 3097
|
||||
"limit": 3089
|
||||
},
|
||||
"ANN002": {
|
||||
"limit": 69
|
||||
},
|
||||
"ANN003": {
|
||||
"limit": 831
|
||||
"limit": 827
|
||||
},
|
||||
"ANN201": {
|
||||
"limit": 2137
|
||||
},
|
||||
"ANN202": {
|
||||
"limit": 941
|
||||
"limit": 939
|
||||
},
|
||||
"ANN204": {
|
||||
"limit": 724
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 130
|
||||
},
|
||||
"ANN401": {
|
||||
"limit": 1848
|
||||
"limit": 1832
|
||||
},
|
||||
"ASYNC230": {
|
||||
"limit": 14
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"LIT001": {
|
||||
"limit": 23349
|
||||
"limit": 23346
|
||||
},
|
||||
"LIT002": {
|
||||
"limit": 27252
|
||||
"limit": 27249
|
||||
},
|
||||
"LIT003": {
|
||||
"limit": 292
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
"limit": 0
|
||||
},
|
||||
"LIT006": {
|
||||
"limit": 1105
|
||||
"limit": 1103
|
||||
},
|
||||
"LIT007": {
|
||||
"limit": 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue