refactor(router): route the router family through RoutingContext and close the outside surface

Strategies, pre-call checks, cooldown and fallback helpers, and the auto, complexity and quality
sub-routers now take a RoutingContext instead of the Router, and read the router only through
registry(), settings(), health() and the context's forwarding methods. Router-free helpers move
from router_utils to litellm_core_utils, outside callers of router state switch to entry points,
and the surface allowlist shrinks to the endpoint methods plus the entry points
This commit is contained in:
narmstrong-berry 2026-10-31 02:02:16 +00:00
parent 5029d027a1
commit 3d33aed3cd
No known key found for this signature in database
156 changed files with 2514 additions and 2082 deletions

View file

@ -1469,11 +1469,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
"AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"),
"HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"),
"get_num_retries_from_retry_policy": (
"litellm.router_utils.get_retry_from_policy",
"litellm.litellm_core_utils.get_retry_from_policy",
"get_num_retries_from_retry_policy",
),
"reset_retry_policy": (
"litellm.router_utils.get_retry_from_policy",
"litellm.litellm_core_utils.get_retry_from_policy",
"reset_retry_policy",
),
"get_coroutine_checker": (

View file

@ -2046,6 +2046,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
AUTH_OBJECTS_TARGET: Final = "auth_objects"
HEALTH_CHECKS_TARGET: Final = "health_checks"
DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)
SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30

View file

@ -66,7 +66,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.table_repositories import EndUserRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
from litellm.router_api import deployment_metadata
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import (
@ -3022,8 +3022,8 @@ class PrometheusLogger(CustomLogger):
).get("custom_llm_provider")
if isinstance(provider, str) and provider:
return provider
deployment_metadata: Final = find_deployment_metadata(litellm_params)
return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
picked_deployment: Final = deployment_metadata(litellm_params)
return _deployment_model_provider(picked_deployment.get("deployment") if picked_deployment else None)
@staticmethod
def _extract_api_provider_from_exception(exception: Exception) -> str | None:
@ -5209,7 +5209,7 @@ def _tag_matches_wildcard_configured_pattern(tags: Sequence[str], configured_tag
"""
import re
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
pattern_router: Final = PatternMatchRouter()
regex_pattern: Final = pattern_router.pattern_to_regex(configured_tag)

View file

@ -3,13 +3,13 @@ from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
)
from litellm.litellm_core_utils.core_helpers import (
filter_internal_params,
safe_deep_copy,
)
from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
)
from .asyncify import run_async_function

View file

@ -119,7 +119,7 @@ def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segmen
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
provider_prefix: Final = f"{custom_llm_provider}/"
segments: Final = wildcard.removeprefix(provider_prefix).split("/")

View file

@ -12,13 +12,13 @@ from datetime import date
from typing import Final, Generic, TypeAlias, TypeVar
from litellm.litellm_core_utils.ptu_pricing import parsed_ptu_shares, ptu_terms
from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, team_may_use_deployment
from litellm.llms.azure.ptu_capacity import (
PTUCapacity,
deployment_ptu_capacity,
is_azure_deployment,
is_str_keyed_mapping,
)
from litellm.router_utils.common_utils import get_request_team_id, team_may_use_deployment
_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
WildcardRoute: TypeAlias = Callable[[str], Sequence[Mapping[str, object]]]

View file

@ -56,9 +56,9 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,

View file

@ -19,6 +19,13 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
strip_provider_specific_fields_from_anthropic_messages,
@ -31,13 +38,6 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.router_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
from litellm.types.llms.anthropic_messages.anthropic_response import (

View file

@ -22,7 +22,7 @@ import litellm
import litellm.constants as _c
from litellm.litellm_core_utils.url_utils import validate_url
from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure
from litellm.router_api import mark_advisor_orchestration_failure
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,

View file

@ -214,7 +214,7 @@ def normalize_reasoning_effort_value(
if chain is None:
return effort
from litellm.router_utils.reasoning_effort_capability import (
from litellm.litellm_core_utils.reasoning_effort_capability import (
deployment_rounds_reasoning_effort,
resolve_supported_reasoning_efforts,
)

View file

@ -4,8 +4,8 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.secret_managers.main import get_secret_str
SUPPORTED_OPENAI_PARAMS: Final = (

View file

@ -9,7 +9,7 @@ from pydantic import TypeAdapter
from litellm.types.llms.openai import AllMessageValues
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
CompactionProtocol: TypeAlias = Literal["chat", "messages"]
_MAPPING: Final = TypeAdapter(Mapping[str, object])
@ -32,7 +32,9 @@ def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompac
return compaction if compaction.supports_native_compaction(params) else None
async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]:
async def dispatch(
router: RoutingContext, protocol: CompactionProtocol, payload: Mapping[str, object]
) -> Mapping[str, object]:
if protocol == "messages":
return _MAPPING.validate_python(
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
@ -45,4 +47,4 @@ async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mappin
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
),
)
return _MAPPING.validate_python(response.model_dump())
return _MAPPING.validate_python(response.model_dump()) # pyright: ignore[reportAttributeAccessIssue] # stream=False: acompletion returns a ModelResponse

View file

@ -25,16 +25,16 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_internal_message_fields,
strip_none_values_from_message,
)
from litellm.litellm_core_utils.reasoning_effort_capability import (
declared_reasoning_efforts_for_model,
nearest_declared_reasoning_effort,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from litellm.router_utils.reasoning_effort_capability import (
declared_reasoning_efforts_for_model,
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import (
MistralCacheKeyParams,

View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
@ -54,7 +55,6 @@ from litellm.llms.oci.common_utils import (
sign_oci_request,
validate_oci_environment,
)
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.oci import (
CohereChatRequest,
OCIChatRequestPayload,

View file

@ -18,7 +18,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema

View file

@ -113,6 +113,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
messages_without_pdf_file_parts,
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.reasoning_effort_capability import (
model_group_from_metadata,
reasoning_effort_needs_allowing,
resolve_chat_reasoning,
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
@ -144,11 +149,6 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.realtime_api.main import realtime_health_check
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.router_utils.reasoning_effort_capability import (
model_group_from_metadata,
reasoning_effort_needs_allowing,
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (

View file

@ -167,6 +167,7 @@ from litellm.repositories.table_repositories import (
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.router import Router
from litellm.router_api import RegistryReader
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
@ -525,7 +526,7 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None:
)
def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
def _get_router_zero_cost_cache(llm_router: RegistryReader) -> dict[str, bool] | None:
"""
Return the router's per-instance zero-cost cache, or ``None`` for objects
that don't expose one (e.g. ``MagicMock`` stand-ins in unit tests).
@ -541,7 +542,7 @@ def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
return cache if isinstance(cache, dict) else None
def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) -> bool:
def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader | None) -> bool:
"""
Check if a model has zero cost (no configured pricing).
@ -667,7 +668,7 @@ _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({})
_TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True})
def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[DeploymentTypedDict]:
def _deployments_served_for(model: str, llm_router: RegistryReader) -> Sequence[DeploymentTypedDict]:
"""
The deployments a request to ``model`` routes to, for the gates that waive budget checks.
@ -4842,7 +4843,7 @@ async def get_skill_names_from_access_groups(
def _resolve_all_team_model_sentinel_for_auth_check(
models: list[str],
llm_router: Router | None,
llm_router: RegistryReader | None,
team_id: str | None,
) -> list[str]:
if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
@ -4856,7 +4857,7 @@ def _resolve_all_team_model_sentinel_for_auth_check(
def check_model_access_helper(
model: str,
llm_router: Router | None,
llm_router: RegistryReader | None,
models: list[str],
team_model_aliases: dict[str, str] | None = None,
team_id: str | None = None,
@ -4907,7 +4908,7 @@ _check_model_access_helper: Final = check_model_access_helper
def _owning_team_names_for_team_deployment(
model: str,
llm_router: Router | None,
llm_router: RegistryReader | None,
team_id: str | None,
team_model_aliases: Mapping[str, str] | None,
) -> tuple[str, ...]:
@ -4925,7 +4926,7 @@ def _owning_team_names_for_team_deployment(
def _can_object_call_model(
model: str | list[str],
llm_router: Router | None,
llm_router: RegistryReader | None,
models: list[str],
team_model_aliases: dict[str, str] | None = None,
team_id: str | None = None,
@ -5052,7 +5053,7 @@ def _resolve_team_alias(
def _live_team_alias_target(
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: Router | None
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: RegistryReader | None
) -> str:
target: Final = team_model_aliases.get(model)
if target is None:
@ -5539,7 +5540,7 @@ async def can_key_call_model(
model: str | list[str],
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
llm_router: RegistryReader | None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
@ -5618,7 +5619,7 @@ async def can_key_call_resolved_model(
model: str,
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
llm_router: RegistryReader | None,
) -> None:
from litellm.proxy.proxy_server import (
prisma_client,
@ -5766,7 +5767,7 @@ def can_org_access_model(
async def can_team_access_model(
model: str | list[str],
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
llm_router: RegistryReader | None,
team_model_aliases: dict[str, str] | None = None,
key_model_aliases: Mapping[str, str] | None = None,
prisma_client: DatabaseClient | None = None,
@ -5869,7 +5870,7 @@ async def key_access_group_grants_model(
model: str | list[str],
valid_token: UserAPIKeyAuth | None,
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
llm_router: RegistryReader | None,
key_model_aliases: Mapping[str, str] | None = None,
) -> bool:
"""
@ -5905,7 +5906,7 @@ _key_access_group_grants_model: Final = key_access_group_grants_model
def can_project_access_model(
model: str | list[str],
project_object: LiteLLM_ProjectTable,
llm_router: Router | None,
llm_router: RegistryReader | None,
key_model_aliases: Mapping[str, str] | None = None,
) -> Literal[True]:
"""
@ -5926,7 +5927,7 @@ def can_project_access_model(
def can_customer_access_model(
model: str | list[str],
end_user_object: LiteLLM_EndUserTable,
llm_router: Router | None,
llm_router: RegistryReader | None,
valid_token: UserAPIKeyAuth | None,
) -> Literal[True]:
team_model_aliases: Final = team_model_aliases_for_auth_check(valid_token) if valid_token is not None else None
@ -6089,7 +6090,7 @@ async def _team_grants_personal_key_model(
model: str,
team_object: LiteLLM_TeamTableCachedObj,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -6127,7 +6128,7 @@ async def _loaded_teams_allow_model(
model: str,
teams: tuple[LiteLLM_TeamTableCachedObj | None, ...],
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -6393,7 +6394,7 @@ async def can_personal_key_call_model_via_teams(
model: str | Sequence[str],
user_object: LiteLLM_UserTable | None,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -7185,7 +7186,7 @@ async def check_team_member_model_access(
model: str | list[str],
team_object: LiteLLM_TeamTable,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: Optional["PrismaClient"],
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,

View file

@ -44,7 +44,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
_is_model_cost_zero, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
is_model_cost_zero, # pyright: ignore[reportPrivateUsage] # the zero-cost predicate the auth-time budget checks use; no public equivalent
)
from litellm.router import Router
from litellm.router_api import RegistryReader, RoutingContext
from litellm.types.llms.base import LiteLLMBaseModel
@ -118,7 +118,7 @@ async def is_token_within_budget_for_model(
*,
model: str,
valid_token: UserAPIKeyAuth,
llm_router: Router,
llm_router: RegistryReader,
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None,
) -> bool:
"""
@ -187,19 +187,19 @@ class RouterFallbackBudgetCheck:
is_enforced: Callable[[], bool]
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
if not self.is_enforced():
return True
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
budget_models: Final = tuple(dict.fromkeys((model, llm_router.registry().resolve(model))))
budget_models: Final = tuple(dict.fromkeys((model, ctx.registry().resolve(model))))
for budget_model in budget_models:
try:
if not await is_token_within_budget_for_model(
model=budget_model,
valid_token=valid_token,
llm_router=llm_router,
llm_router=ctx,
team_member_model_budget_limiter=self.team_member_model_budget_limiter,
):
return False

View file

@ -18,7 +18,7 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool
from litellm.router import Router
from litellm.router_api import RegistryReader, RoutingContext
from litellm.search import asearch
from litellm.types.llms.base import LiteLLMBaseModel
@ -31,7 +31,7 @@ class _FallbackAccessSettings(LiteLLMBaseModel):
enforce_fallback_model_access: bool = True
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: RegistryReader) -> bool:
try:
await can_key_call_resolved_model(
model=model,
@ -98,7 +98,7 @@ class RouterFallbackAccessCheck:
is_enforced: Callable[[], bool]
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
@ -106,7 +106,7 @@ class RouterFallbackAccessCheck:
return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token)
if not self.is_enforced():
return True
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx)
router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings)

View file

@ -14,13 +14,13 @@ from typing import Final, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.responses.main import strip_responses_routing_prefix
from litellm.router import Router
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
@ -708,7 +708,7 @@ def get_all_fallbacks(
try:
# Use existing function to get fallback model group
fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks_config, model_group=model)
fallback_model_group, _ = fallback_chain(fallbacks=fallbacks_config, model_group=model)
if fallback_model_group is None:
return []

View file

@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
from litellm.models.end_user import LiteLLM_EndUserTable
from litellm.proxy._types import *
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
@ -182,13 +183,7 @@ from litellm.proxy.utils import (
from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.router_utils.fallback_event_handlers import (
PROVIDER_SCOPED_RESOURCE_KEYS,
fallback_lookup_groups,
fallbacks_disabled_for_request,
get_fallback_model_group_for_lookup_groups,
)
from litellm.router_api import fallbacks_disabled, references_provider_scoped_resource, request_fallback_chain
from litellm.secret_managers.main import get_secret_bool
from litellm.types.services import ServiceTypes
@ -811,11 +806,7 @@ async def _check_team_member_model_budget_for_model(
model=model_name,
)
except litellm.BudgetExceededError:
if (
request_data.get("model") != model_name
or llm_router is None
or fallbacks_disabled_for_request(kwargs=request_data)
):
if request_data.get("model") != model_name or llm_router is None or fallbacks_disabled(request_data):
raise
fallback_succeeded: Final = await _try_team_member_model_budget_fallback(
user_id=user_id,
@ -857,9 +848,9 @@ async def _try_team_member_model_budget_fallback(
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> bool:
if request.headers.get("content-type", "").lower().startswith("multipart/form-data") or any(
request_data.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS
):
if request.headers.get("content-type", "").lower().startswith(
"multipart/form-data"
) or references_provider_scoped_resource(request_data):
return False
from litellm.proxy import proxy_server
@ -896,10 +887,7 @@ async def _try_team_member_model_budget_fallback(
if router_fallbacks is None:
return False
fallback_models, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=list(router_fallbacks),
lookup_groups=fallback_lookup_groups(kwargs=request_data, model_group=model_name),
)
fallback_models, _ = request_fallback_chain(list(router_fallbacks), request_data, model_name)
fallback_targets: Final[Sequence[object]] = fallback_models or ()
fallback_candidates: Final[tuple[str, ...]] = tuple(
model for target in fallback_targets if (model := _budget_fallback_target_model(target)) is not None

View file

@ -19,8 +19,8 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
build_environment_report,
)
from litellm.litellm_core_utils.routing_groups import VALID_ROUTING_STRATEGIES
from litellm.proxy._types import ConfigGeneralSettings
from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES
from litellm.types.caching import LiteLLMCacheType
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations
from litellm.types.secret_managers.main import KeyManagementSystem

View file

@ -53,6 +53,7 @@ from litellm.constants import (
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
@ -81,6 +82,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
get_response_headers,
)
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.served_output_texts import (
record_served_output_texts,
@ -146,8 +148,6 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.types.extraction import ExtractionResponse
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
@ -2565,7 +2565,7 @@ class ProxyBaseLLMRequestProcessing:
aliased_model: Final = self.data.get("model")
if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
self.data.update(
llm_router.priority_header_update(
llm_router.health().priority_header_fields(
model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
)
)
@ -2831,12 +2831,9 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
def _resolve_fallback_models(model: str, fallbacks: list) -> list[str] | None:
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_api import fallback_chain
fallback_model_group, generic_fallback_idx = get_fallback_model_group(
fallbacks=fallbacks,
model_group=model,
)
fallback_model_group, generic_fallback_idx = fallback_chain(fallbacks, model)
if fallback_model_group is not None:
return fallback_model_group
return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None
@ -4190,8 +4187,8 @@ class ProxyBaseLLMRequestProcessing:
HiddenParamsAsyncIteratorWrapper, so class sniffing runs on the
unwrapped inner iterator.
"""
from litellm.litellm_core_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
unwrapped: Final = response.inner if isinstance(response, HiddenParamsAsyncIteratorWrapper) else response

View file

@ -16,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_f
from litellm.llms.anthropic.cache_aware_routing import AnthropicCacheRouting, TokenCounter
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
from litellm.router_deprecated import renamed_router_param
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import Deployment, PreRoutingHookResponse
@ -23,6 +24,7 @@ from litellm.types.utils import StandardLoggingRoutingDecision
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
_MESSAGES: Final = TypeAdapter(list[Mapping[str, object]])
_MAPPING: Final = TypeAdapter(Mapping[str, object])
@ -231,7 +233,7 @@ async def select_cached_model(
async def _choose_cached_model(
*,
router: Router,
ctx: RoutingContext,
config: ComplexityRouterConfig,
params_for_model: Callable[[str, str], Mapping[str, object]],
response: PreRoutingHookResponse | None,
@ -249,8 +251,10 @@ async def _choose_cached_model(
)
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
router: Final = proxy_server.llm_router
if (
proxy_server.llm_router is not router
router is None
or not ctx.belongs_to(router)
or router.registry().plugins("routing")
or has_request_transforms()
or compaction_pending(request_kwargs)
@ -318,9 +322,10 @@ async def _choose_cached_model(
)
@renamed_router_param("router")
async def choose_cached_model(
*,
router: Router,
ctx: RoutingContext,
config: ComplexityRouterConfig,
params_for_model: Callable[[str, str], Mapping[str, object]],
response: PreRoutingHookResponse | None,
@ -332,7 +337,7 @@ async def choose_cached_model(
try:
return await asyncio.wait_for(
_choose_cached_model(
router=router,
ctx=ctx,
config=config,
params_for_model=params_for_model,
response=response,

View file

@ -12,10 +12,10 @@ from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # narrows untyped credential values at the decrypt boundary
import litellm
from litellm.litellm_core_utils.clientside_credential_handler import clientside_credential_keys
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.proxy.utils import PrismaClient
from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.router_utils.clientside_credential_handler import clientside_credential_keys
from litellm.types.router import (
GenericLiteLLMParams,
server_owned_wif_fields_present,

View file

@ -21,7 +21,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import CLAUDE_CODE_CLIENT, CLAUDE_CODE_PICKER_PATTERN, GATEWAY_CLIENT_HEADER
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
if TYPE_CHECKING:
from litellm.router import Router

View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
has_tool_with_name,
)
from litellm.litellm_core_utils.provider_affinity import get_caller_session_id
from litellm.litellm_core_utils.routing_common_utils import request_user_api_key_auth
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
httpxSpecialProvider,
@ -40,7 +41,6 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import (
merge_rewritten_text_parts,
)
from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
from litellm.router_utils.common_utils import request_user_api_key_auth
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks, Mode
from litellm.types.integrations.custom_logger import (

View file

@ -14,6 +14,7 @@ from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD
from litellm.router_api import RoutingContext
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
@ -134,7 +135,7 @@ class SemanticGuardRouteLoader:
)
encoder: Final = LiteLLMRouterEncoder(
litellm_router_instance=litellm_router,
ctx=RoutingContext(litellm_router),
model_name=embedding_model,
score_threshold=global_threshold,
)

View file

@ -27,12 +27,12 @@ from litellm.constants import (
HEALTH_CHECK_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependency,
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
# Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy

View file

@ -13,10 +13,10 @@ from litellm.caching.redis_cache import RedisCache
from litellm.constants import (
DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL,
DEFAULT_SHARED_HEALTH_CHECK_TTL,
HEALTH_CHECKS_TARGET,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.health_check import perform_health_check
from litellm.router_utils.health_state_cache import HEALTH_CHECKS_TARGET
if TYPE_CHECKING:
from litellm.router import Router

View file

@ -26,7 +26,12 @@ from litellm.integrations.SlackAlerting.ms_teams import (
build_ms_teams_payload,
get_ms_teams_webhook_url,
)
from litellm.litellm_core_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import (
AlertType,
@ -94,11 +99,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
from litellm.router import Router
from litellm.router_api import deployment_usable_by_team
from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
from litellm.types.router import reject_server_owned_wif_fields

View file

@ -15,6 +15,10 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import (
ProxyRateLimitError,
@ -34,10 +38,6 @@ from litellm.proxy.hooks.rate_limiter_utils import (
resolve_llm_provider_for_rate_limit,
)
from litellm.proxy.utils import InternalUsageCache
from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.types.router import ModelGroupInfo
from litellm.types.utils import CallTypesLiteral, LLMResponseTypes

View file

@ -204,7 +204,7 @@ def _team_public_names_by_group(router: "Router", team_id: str | None) -> Mappin
groups_by_name: Final = MappingProxyType(
{
name: model_access_groups_of(router=router, model=name, team_id=team_id)
for name in router.team_public_model_names_of(team_id)
for name in router.registry().names(kind="team_public", team_id=team_id)
}
)
return MappingProxyType(

View file

@ -49,6 +49,10 @@ from litellm.constants import (
)
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -56,6 +60,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
split_media_for_tpm,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
@ -86,12 +92,6 @@ from litellm.proxy.spend_tracking.input_tokens import (
openai_chat_pdf_file_tokens,
)
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.ptu_shares import PTUTeamCeiling
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.extraction import ExtractionResponse
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
@ -3561,9 +3561,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
Returns True if any deployment has failures in the current minute.
"""
from litellm.proxy.proxy_server import llm_router
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
get_deployment_failures_for_current_minute,
)
if llm_router is None:
return False
@ -3581,10 +3578,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
continue
# Use router's existing failure tracking
failure_count = get_deployment_failures_for_current_minute(
litellm_router_instance=llm_router,
deployment_id=deployment_id,
)
failure_count = llm_router.health().failures_this_minute(deployment_id)
if failure_count > DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE:
verbose_proxy_logger.debug(

View file

@ -3291,7 +3291,7 @@ def _fallback_edges(
and its provider-stripped suffix, plus the router matcher run over the fuzzy
entries that have not fired yet (tracked in ``fired_fuzzy`` by
``(list_index, entry_index)``), so each entry's targets are expanded once."""
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_api import fallback_chain
stripped: Final = model_group.split("/", 1)[1] if "/" in model_group else None
targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets
@ -3302,7 +3302,7 @@ def _fallback_edges(
pending = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy)
if not pending:
continue
resolved = get_fallback_model_group(fallbacks=[fuzzy[index] for index in pending], model_group=model_group)[0]
resolved = fallback_chain([fuzzy[index] for index in pending], model_group)[0]
values = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else [])
targets.update(name for value in values if (name := _fallback_target_name(value)) is not None)
if resolved is not None:
@ -3489,7 +3489,7 @@ async def _resolve_user_provider_credentials_for_request(
status_code=400,
detail=(f"fallback lists in the request body cannot exceed {_FALLBACK_DISCOVERY_LIMIT} entries"),
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
# Discovery must see the group the router actually routes to, so include the
# post-alias name from every alias map that may still rewrite data["model"]

View file

@ -60,6 +60,7 @@ from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.router_api import RoutingContext
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependencyRole,
classify_strategy_router_model,
@ -614,7 +615,7 @@ async def preview_auto_router_routing(
try:
complexity_strategy: Final = ComplexityRouter(
model_name=resolved.router_name,
litellm_router_instance=llm_router,
ctx=RoutingContext(llm_router),
complexity_router_config=resolved.complexity_router_config.model_dump(exclude_none=True),
default_model=resolved.complexity_router_config.resolve_default_model(resolved.default_model),
derive_savings_baseline=False,

View file

@ -12,14 +12,14 @@ from types import MappingProxyType
from typing import Final
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
from litellm.router import Router
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.ptu_shares import (
from litellm.litellm_core_utils.ptu_shares import (
model_group_ptu_capacity,
routed_deployments,
team_servable_deployments,
)
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
from litellm.router import Router
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailySpendData,
MetricWithMetadata,

View file

@ -9,10 +9,10 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
from litellm.litellm_core_utils.routing_common_utils import provider_for_generic_call
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.common_utils import provider_for_generic_call
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:

View file

@ -5,8 +5,8 @@ from typing import Final
from fastapi import HTTPException
from litellm.litellm_core_utils.ptu_shares import deployments_shared_with
from litellm.router import Router
from litellm.router_utils.ptu_shares import deployments_shared_with
def raise_if_teams_hold_ptu_shares(team_ids: Sequence[str], llm_router: Router | None) -> None:

View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager
from litellm.proxy.mcp_server.faults import iter_exception_tree
from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR
from litellm.router_api import RoutingContext
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
@ -172,7 +173,7 @@ class SemanticMCPToolFilter:
self.tool_router = SemanticRouter(
routes=routes,
encoder=LiteLLMRouterEncoder(
litellm_router_instance=self.router_instance,
ctx=RoutingContext(self.router_instance),
model_name=self.embedding_model,
score_threshold=self.similarity_threshold,
),
@ -247,7 +248,7 @@ class SemanticMCPToolFilter:
router: Final = SemanticRouter(
routes=[],
encoder=LiteLLMRouterEncoder(
litellm_router_instance=self.router_instance,
ctx=RoutingContext(self.router_instance),
model_name=self.embedding_model,
score_threshold=self.similarity_threshold,
),

View file

@ -1786,7 +1786,7 @@ def _routed_model_name_on_provider(
def _stored_model_name_routes_to_provider(
llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
) -> bool:
routing_group: Final = llm_router.get_routing_group(model_name)
routing_group: Final = llm_router.registry().routing_group(model_name)
routed_model_names: Final = (
tuple(routing_group.models) if routing_group is not None else (llm_router.registry().resolve(model_name),)
)

View file

@ -7,6 +7,7 @@ from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
add_policy_sources_to_metadata,
@ -18,7 +19,6 @@ from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.types.proxy.policy_engine import PolicyMatchContext
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline

View file

@ -89,6 +89,11 @@ from litellm.constants import (
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
)
from litellm.litellm_core_utils.access_windows import access_windows_config_error
from litellm.litellm_core_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.bug_report import (
allowlisted,
@ -100,6 +105,8 @@ from litellm.litellm_core_utils.litellm_logging import (
initialize_sentry,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.routing_groups import parse_routing_groups
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.token_counter import (
@ -189,13 +196,6 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
snapshot_tuning_baselines,
tuning_limit_violation,
)
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.routing_groups import parse_routing_groups
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
@ -999,7 +999,7 @@ from litellm.router import (
LiteLLM_Params,
ModelGroupInfo,
)
from litellm.router_api import ChangeSet, deployment_id
from litellm.router_api import ChangeSet, deployment_id, router_init_args
from litellm.scheduler import FlowItem, Scheduler
from litellm.secret_managers.aws_secret_manager import load_aws_kms
from litellm.secret_managers.google_kms import load_google_kms
@ -4660,10 +4660,6 @@ def _write_health_state_to_router_cache(
is on.
"""
from litellm.proxy.health_check import build_deployment_health_states
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
increment_deployment_failures_for_current_minute,
)
_exceptions: Final[dict] = exceptions_by_model_id or {}
@ -4715,18 +4711,7 @@ def _write_health_state_to_router_cache(
):
continue
increment_deployment_failures_for_current_minute(
litellm_router_instance=llm_router,
deployment_id=model_id,
)
set_cooldown_deployments(
litellm_router_instance=llm_router,
original_exception=original_exception,
exception_status=exception_status,
deployment=model_id,
time_to_cooldown=router_settings.cooldown_time,
)
llm_router.health().record_failure(model_id, original_exception, exception_status)
except Exception as e:
verbose_proxy_logger.warning("Failed to write health state to router cache: %s", str(e))
@ -7370,9 +7355,7 @@ class ProxyConfig:
router_settings: Final = config.get("router_settings", None)
if router_settings and isinstance(router_settings, dict):
available_args: Final = [
x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
]
available_args: Final = [x for x in router_init_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG]
for k, v in router_settings.items():
if k in available_args:
@ -9596,7 +9579,6 @@ class ProxyConfig:
SearchToolRegistry,
keep_loaded_search_tools_that_do_not_decrypt,
)
from litellm.router_utils.search_api_router import SearchAPIRouter
try:
db_search_tools: Final = keep_loaded_search_tools_that_do_not_decrypt(
@ -9620,7 +9602,21 @@ class ProxyConfig:
)
if llm_router is not None:
await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools)
llm_router.apply(
ChangeSet(
assign={
"search_tools": [
{
"search_tool_id": tool.get("search_tool_id"),
"search_tool_name": tool.get("search_tool_name"),
"litellm_params": tool.get("litellm_params", {}),
"search_tool_info": tool.get("search_tool_info"),
}
for tool in search_tools
]
}
)
)
verbose_proxy_logger.info("Successfully loaded %s search tool(s) into router", len(search_tools))
else:
verbose_proxy_logger.debug(
@ -16152,7 +16148,7 @@ def _enrich_model_info_with_litellm_data(
if debug is True:
_openai_client = "None"
if llm_router is not None:
_openai_client = llm_router._get_client(deployment=model, kwargs={}, client_type="async") or "None"
_openai_client = llm_router.registry().client(model, client_type="async") or "None"
else:
_openai_client = "llm_router_is_None"
openai_client: Final = str(_openai_client)
@ -18602,7 +18598,7 @@ async def async_queue_request(
router: Final = llm_router
data.update(
router.priority_header_update(
router.health().priority_header_fields(
model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
)
)
@ -19751,7 +19747,7 @@ async def update_config(
raw_router_settings: Final = request_body.get("router_settings")
if isinstance(raw_router_settings, dict):
supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | (
frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
frozenset(router_init_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
)
unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings)
if unsupported_router_settings:

View file

@ -16,8 +16,9 @@ from pydantic import BeforeValidator, TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.router_utils.common_utils import is_proxy_admin_request
from litellm.router_api import router_init_args
# Client-supplied params that make the router or the call path fabricate a
# failure or a delay instead of calling the provider. The ``mock_testing_*``
@ -93,7 +94,7 @@ async def _route_user_config_request(data: dict, route_type: str):
# Filter router_config to only include valid Router.__init__ arguments
# This prevents TypeError when invalid parameters are stored in the database
valid_args: Final = litellm.Router.get_valid_args()
valid_args: Final = router_init_args()
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)

View file

@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.litellm_core_utils.token_counter import (
deployment_openai_model,
pdf_page_pricing,
@ -31,7 +32,6 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
billable_input_since_newest_compaction,
holds_compaction_summary,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.token_counter import (
@ -162,7 +162,7 @@ def models_counted_for(model: str, llm_router: Router | None, team_id: str | Non
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
registry: Final = llm_router.registry()
resolved: Final = registry.resolve(model, kind="alias") or model
routing_group: Final = llm_router.get_routing_group(resolved)
routing_group: Final = registry.routing_group(resolved)
members: Final = routing_group.models if routing_group is not None else (resolved,)
named: Final = tuple(
chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members)

View file

@ -160,10 +160,12 @@ from litellm.litellm_core_utils.core_helpers import (
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.deployment_pricing import pricing_models
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.served_output_texts import (
@ -278,8 +280,6 @@ from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import (
VerificationTokenRepository,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.deployment_pricing import pricing_models
from litellm.secret_managers.main import str_to_bool
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
from litellm.types.llms.openai import ResponsesAPIResponse

View file

@ -42,6 +42,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
update_responses_tools_with_model_file_ids,
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.reasoning_effort_capability import (
deployment_reasoning_effort_levels,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
@ -63,11 +68,6 @@ from litellm.responses.mcp.round_billing import (
)
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.reasoning_effort_capability import (
deployment_reasoning_effort_levels,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.types.llms.openai import (
PromptObject,
Reasoning,

View file

@ -106,6 +106,20 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
from litellm.litellm_core_utils.access_windows import access_windows_config_error, filter_reserved_deployments
from litellm.litellm_core_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
add_retry_headers_to_response,
apply_quality_router_decision_headers,
apply_remaining_usage_headers,
apply_response_model_id,
complexity_router_decision_headers,
ensure_response_additional_headers,
get_hidden_params_dict,
prepare_response_for_header_attachment,
replace_complexity_router_headers,
response_total_token_count,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.batch_utils import (
batch_create_routing,
@ -116,6 +130,10 @@ from litellm.litellm_core_utils.batch_utils import (
should_replace_model_in_jsonl,
without_deployments_that_cannot_batch,
)
from litellm.litellm_core_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
is_clientside_credential,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
@ -127,12 +145,17 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
inferred_provider,
is_registered_custom_provider,
resolve_model_and_provider_for_metadata,
)
from litellm.litellm_core_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
)
from litellm.litellm_core_utils.get_retry_from_policy import resolve_retry_policy
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR,
)
@ -142,6 +165,7 @@ from litellm.litellm_core_utils.hidden_params import (
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
declares_ptu,
@ -152,10 +176,52 @@ from litellm.litellm_core_utils.ptu_pricing import (
ptu_terms,
zeroed_ptu_pricing,
)
from litellm.litellm_core_utils.ptu_shares import (
PTUTeamCeiling,
filter_ptu_shared_deployments,
ptu_capacity_warning,
ptu_share_teams,
share_holder_request_kwargs,
team_ptu_ceiling,
)
from litellm.litellm_core_utils.reasoning_effort_capability import (
deployment_is_catalog_mapped,
intersect_supported_reasoning_efforts,
resolve_supported_reasoning_efforts,
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.routing_common_utils import (
MID_STREAM_CONTINUATION_KWARG,
as_retry_skipped_deployment_ids,
deployment_model_id_pin,
deployments_the_caller_may_use,
filter_team_based_models,
filter_web_search_deployments,
format_content_policy_fallback_hint,
format_content_policy_no_fallback_note,
format_fallback_outcome_message,
format_no_fallback_group_message,
get_request_team_id,
is_proxy_admin_request,
provider_for_generic_call,
request_user_api_key_auth,
resolve_model_group_alias,
team_may_use_deployment,
truncate_fallback_error_detail,
warn_on_ignored_model_capability_flags,
warn_on_provider_credential_mismatch,
with_admin_api_key_secret_reference,
without_model_capability_flags,
without_router_only_kwargs,
)
from litellm.litellm_core_utils.routing_groups import (
apply_routing_group_priority,
parse_routing_groups,
validate_routing_strategy,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.sensitive_data_masker import (
SensitiveDataMasker,
@ -196,6 +262,7 @@ from litellm.router_api import (
aapply_changes,
apply_changes,
)
from litellm.router_api import RoutingContext as RouterContext
from litellm.router_api import deployment_id as _deployment_id
from litellm.router_deprecated import DeprecatedRouterMembers
from litellm.router_host import router_host
@ -233,49 +300,7 @@ from litellm.router_strategy.tag_based_routing import (
is_valid_deployment_tag,
tags_with_serving_deployment_tags,
)
from litellm.router_utils.access_windows import access_windows_config_error, filter_reserved_deployments
from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
add_retry_headers_to_response,
apply_quality_router_decision_headers,
apply_remaining_usage_headers,
apply_response_model_id,
complexity_router_decision_headers,
ensure_response_additional_headers,
get_hidden_params_dict,
prepare_response_for_header_attachment,
replace_complexity_router_headers,
response_total_token_count,
)
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
from litellm.router_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
is_clientside_credential,
)
from litellm.router_utils.common_utils import (
MID_STREAM_CONTINUATION_KWARG,
as_retry_skipped_deployment_ids,
deployment_model_id_pin,
deployments_the_caller_may_use,
filter_team_based_models,
filter_web_search_deployments,
format_content_policy_fallback_hint,
format_content_policy_no_fallback_note,
format_fallback_outcome_message,
format_no_fallback_group_message,
get_request_team_id,
is_proxy_admin_request,
provider_for_generic_call,
request_user_api_key_auth,
resolve_model_group_alias,
team_may_use_deployment,
truncate_fallback_error_detail,
warn_on_ignored_model_capability_flags,
warn_on_provider_credential_mismatch,
with_admin_api_key_secret_reference,
without_model_capability_flags,
without_router_only_kwargs,
)
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.cooldown_handlers import (
DEFAULT_COOLDOWN_TIME_SECONDS,
@ -292,7 +317,6 @@ from litellm.router_utils.cooldown_handlers import (
mark_deployment_selection_failure,
set_cooldown_deployments,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.router_utils.embedding_batch_size_check import (
async_log_rejection_before_provider_call,
clear_rejected_before_provider_call,
@ -337,10 +361,6 @@ from litellm.router_utils.fallback_event_handlers import (
routed_deployment_id,
run_async_fallback,
)
from litellm.router_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
)
from litellm.router_utils.get_retry_from_policy import resolve_retry_policy
from litellm.router_utils.handle_error import (
async_raise_no_deployment_exception,
send_llm_exception_alert,
@ -392,19 +412,6 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
PromptCachingDeploymentCheck,
)
from litellm.router_utils.ptu_shares import (
PTUTeamCeiling,
filter_ptu_shared_deployments,
ptu_capacity_warning,
ptu_share_teams,
share_holder_request_kwargs,
team_ptu_ceiling,
)
from litellm.router_utils.reasoning_effort_capability import (
deployment_is_catalog_mapped,
intersect_supported_reasoning_efforts,
resolve_supported_reasoning_efforts,
)
from litellm.router_utils.request_priority import (
InvalidPriority,
parse_default_priority,
@ -420,11 +427,6 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import (
increment_deployment_failures_for_current_minute,
increment_deployment_successes_for_current_minute,
)
from litellm.router_utils.routing_groups import (
apply_routing_group_priority,
parse_routing_groups,
validate_routing_strategy,
)
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
from litellm.scheduler import FlowItem, Scheduler
@ -520,8 +522,6 @@ from litellm.utils import (
token_counter,
)
from .router_utils.pattern_match_deployments import PatternMatchRouter
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -1610,6 +1610,7 @@ class Router(DeprecatedRouterMembers):
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
```
"""
self._core_ctx: Final = RouterContext(self)
self.set_verbose = set_verbose
self.ignore_invalid_deployments = ignore_invalid_deployments
@ -1885,7 +1886,7 @@ class Router(DeprecatedRouterMembers):
if self._normalize_strategy(routing_strategy) == "lar1":
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
apply_lar1_routing_strategy(self, routing_strategy_args)
apply_lar1_routing_strategy(self._core_ctx, routing_strategy_args)
else:
self.routing_strategy_init(
routing_strategy=routing_strategy,
@ -2259,7 +2260,7 @@ class Router(DeprecatedRouterMembers):
if strategy == "lar1":
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
apply_lar1_routing_strategy(self, self.routing_strategy_args)
apply_lar1_routing_strategy(self._core_ctx, self.routing_strategy_args)
return
attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "")
@ -2584,7 +2585,7 @@ class Router(DeprecatedRouterMembers):
)
deployments: Final = self._core_get_model_list(model_name=model)
if deployments:
RoutingPrefetch.arm(self, usage_selector, deployments)
RoutingPrefetch.arm(self._core_ctx, usage_selector, deployments)
except Exception as e: # noqa: BLE001 # a prefetch is an optimisation, never a reason to fail the request
verbose_router_logger.debug(
"routing read prefetch not armed for %s: %s", _without_line_breaks(model), _without_line_breaks(e)
@ -3133,7 +3134,7 @@ class Router(DeprecatedRouterMembers):
break
if existing_ec_callback is not None:
existing_ec_callback.router = self
existing_ec_callback.ctx = self._core_ctx
existing_ec_callback.enable_global_affinity = (
existing_ec_callback.enable_global_affinity or enable_global_affinity
)
@ -3141,7 +3142,7 @@ class Router(DeprecatedRouterMembers):
ec_callback = existing_ec_callback
else:
ec_callback = EncryptedContentAffinityCheck(
router=self,
ctx=self._core_ctx,
enable_global_affinity=enable_global_affinity,
model_group_affinity_config=self.model_group_affinity_config,
)
@ -3690,7 +3691,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -4933,7 +4934,7 @@ class Router(DeprecatedRouterMembers):
kwargs=kwargs,
client_type="max_parallel_requests",
)
compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat")
compacted_input: Final = await compact_to_fit(self._core_ctx, deployment, input_kwargs, "chat")
async with contextlib.AsyncExitStack() as deployment_slot:
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
deployment_slot.enter_context(max_parallel_requests_limit)
@ -5396,7 +5397,7 @@ class Router(DeprecatedRouterMembers):
hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
if _is_fallback_hop(hop_kwargs):
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
self,
self._core_ctx,
hop_kwargs.get("input"),
hop_kwargs.get("messages"),
(_FALLBACK_HOP_ADAPTER.validate_python(deployment),),
@ -6002,7 +6003,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6114,7 +6115,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6207,7 +6208,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6269,7 +6270,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6385,7 +6386,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6455,7 +6456,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6514,7 +6515,7 @@ class Router(DeprecatedRouterMembers):
return _OPAQUE_VALUE_ADAPTER.validate_python(
await SearchAPIRouter.async_search_with_fallbacks(
router_instance=self,
litellm_routing_ctx=self._core_ctx,
original_function=original_function,
**kwargs,
)
@ -6528,7 +6529,7 @@ class Router(DeprecatedRouterMembers):
from litellm.router_utils.search_api_router import SearchAPIRouter
return await SearchAPIRouter.async_search_with_fallbacks_helper(
router_instance=self,
litellm_routing_ctx=self._core_ctx,
model=model,
original_generic_function=original_generic_function,
**kwargs,
@ -6646,7 +6647,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -6746,7 +6747,7 @@ class Router(DeprecatedRouterMembers):
response_kwargs["custom_llm_provider"] = custom_llm_provider
compacted_input: Final = await compact_to_fit(
self,
self._core_ctx,
deployment,
response_kwargs,
surface_for_call(generic_function_name),
@ -7624,7 +7625,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -7703,7 +7704,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -7717,7 +7718,7 @@ class Router(DeprecatedRouterMembers):
**kwargs,
) -> OpenAIFileObject:
try:
from litellm.router_utils.common_utils import add_model_file_id_mappings
from litellm.litellm_core_utils.routing_common_utils import add_model_file_id_mappings
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
@ -7943,7 +7944,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -8111,7 +8112,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -8142,7 +8143,7 @@ class Router(DeprecatedRouterMembers):
except Exception as e:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=self,
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -8932,7 +8933,7 @@ class Router(DeprecatedRouterMembers):
# that fails with RouterRateLimitError whenever the "remaining" entries
# are all in cooldown — the inner async_get_healthy_deployments call
# would find an empty list and raise immediately.
cooldown_ids = set(await async_get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=None))
cooldown_ids = set(await async_get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=None))
remaining: Final = (all_ids - cooldown_ids) - excluded
if not remaining:
return None
@ -9012,7 +9013,7 @@ class Router(DeprecatedRouterMembers):
raise e
input_kwargs: Final = {
"litellm_router": self,
"litellm_routing_ctx": self._core_ctx,
"original_exception": original_exception,
**kwargs,
"attempted_targets": fallback_walk,
@ -9228,7 +9229,7 @@ class Router(DeprecatedRouterMembers):
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
fallback_failure_exception_str = truncate_fallback_error_detail(redact_string(str(new_exception)))
cooldown_info: Final = await async_get_cooldown_deployments_with_debug_info(
litellm_router_instance=self,
ctx=self._core_ctx,
parent_otel_span=parent_otel_span,
)
verbose_router_logger.error(
@ -9849,7 +9850,7 @@ class Router(DeprecatedRouterMembers):
# Always track deployment successes for cooldown logic, regardless of TPM/RPM limits
increment_deployment_successes_for_current_minute(
litellm_router_instance=self,
ctx=self._core_ctx,
deployment_id=str(deployment_id),
)
@ -10029,7 +10030,7 @@ class Router(DeprecatedRouterMembers):
if id is not None:
key: Final = increment_deployment_successes_for_current_minute(
litellm_router_instance=self,
ctx=self._core_ctx,
deployment_id=id,
)
return key
@ -10140,11 +10141,11 @@ class Router(DeprecatedRouterMembers):
if deployment_id is None:
return False
increment_deployment_failures_for_current_minute(
litellm_router_instance=self,
ctx=self._core_ctx,
deployment_id=deployment_id,
)
result: Final = set_cooldown_deployments(
litellm_router_instance=self,
ctx=self._core_ctx,
exception_status=exception_status,
original_exception=exception,
deployment=deployment_id,
@ -10450,9 +10451,7 @@ class Router(DeprecatedRouterMembers):
except Exception:
pass
unhealthy_deployments: Final = get_cooldown_deployments(
litellm_router_instance=self, parent_otel_span=parent_otel_span
)
unhealthy_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
unhealthy_set: Final = set(unhealthy_deployments)
healthy_deployments: list = [d for d in _all_deployments if d["model_info"]["id"] not in unhealthy_set]
healthy_deployments = self._filter_blocked_deployments(healthy_deployments)
@ -10480,7 +10479,7 @@ class Router(DeprecatedRouterMembers):
pass
unhealthy_deployments: Final = await async_get_cooldown_deployments(
litellm_router_instance=self, parent_otel_span=parent_otel_span
ctx=self._core_ctx, parent_otel_span=parent_otel_span
)
# Convert to set for O(1) lookup instead of O(n)
unhealthy_deployments_set: Final = set(unhealthy_deployments)
@ -10538,7 +10537,7 @@ class Router(DeprecatedRouterMembers):
)
)
set_cooldown_deployments(
litellm_router_instance=self,
ctx=self._core_ctx,
exception_status=e.status_code,
original_exception=e,
deployment=deployment["model_info"]["id"],
@ -10622,7 +10621,7 @@ class Router(DeprecatedRouterMembers):
)
if selected is None:
no_deployment: Final = await async_raise_no_deployment_exception(
litellm_router_instance=self, model=model_group, parent_otel_span=None
ctx=self._core_ctx, model=model_group, parent_otel_span=None
)
raise litellm.RateLimitError(
message=str(no_deployment),
@ -11053,7 +11052,7 @@ class Router(DeprecatedRouterMembers):
auto_router_config=auto_router_config,
default_model=default_model,
embedding_model=embedding_model,
litellm_router_instance=self,
ctx=self._core_ctx,
max_input_chars=(max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS),
)
self._register_pre_routing_strategy(
@ -11134,7 +11133,7 @@ class Router(DeprecatedRouterMembers):
complexity_router: Final[ComplexityRouter] = ComplexityRouter(
model_name=deployment.model_name,
default_model=default_model,
litellm_router_instance=self,
ctx=self._core_ctx,
complexity_router_config=complexity_router_config,
)
self._register_pre_routing_strategy(
@ -11434,7 +11433,7 @@ class Router(DeprecatedRouterMembers):
quality_router: Final[QualityRouter] = QualityRouter(
model_name=deployment.model_name,
default_model=default_model,
litellm_router_instance=self,
ctx=self._core_ctx,
quality_router_config=quality_router_config,
)
self._register_pre_routing_strategy(
@ -13655,7 +13654,7 @@ class Router(DeprecatedRouterMembers):
while remaining:
try:
picked = await get_deployments_for_tag(
llm_router_instance=self,
ctx=self._core_ctx,
model=model,
healthy_deployments=remaining,
request_kwargs=tag_request,
@ -14538,7 +14537,7 @@ class Router(DeprecatedRouterMembers):
if routing_strategy == "lar1":
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
apply_lar1_routing_strategy(self, kwargs.get("routing_strategy_args"))
apply_lar1_routing_strategy(self._core_ctx, kwargs.get("routing_strategy_args"))
return
self.routing_strategy_init(
routing_strategy=routing_strategy, routing_strategy_args=kwargs.get("routing_strategy_args", {})
@ -14631,7 +14630,7 @@ class Router(DeprecatedRouterMembers):
cache_key = f"{model_id}_max_parallel_requests_client"
client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span)
if client is None:
InitalizeCachedClient.set_max_parallel_requests_client(litellm_router_instance=self, model=deployment)
InitalizeCachedClient.set_max_parallel_requests_client(ctx=self._core_ctx, model=deployment)
client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span)
return client
elif client_type == "async":
@ -15562,10 +15561,10 @@ class Router(DeprecatedRouterMembers):
routing_read_batch: Final = RoutingReadBatch.active()
cooldown_deployments: Final = (
await async_get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
await async_get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
if routing_read_batch is None
else await routing_read_batch.async_get_cooldown_deployments(
litellm_router_instance=self,
ctx=self._core_ctx,
healthy_deployments=healthy_deployments,
parent_otel_span=parent_otel_span,
)
@ -15620,7 +15619,7 @@ class Router(DeprecatedRouterMembers):
)
# check if user wants to do tag based routing
healthy_deployments = await get_deployments_for_tag(
llm_router_instance=self,
ctx=self._core_ctx,
model=model,
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
@ -15699,7 +15698,7 @@ class Router(DeprecatedRouterMembers):
if len(healthy_deployments) == 0:
exception: Final = await async_raise_no_deployment_exception(
litellm_router_instance=self,
ctx=self._core_ctx,
model=model,
parent_otel_span=parent_otel_span,
route_model_ids=await self._async_routable_cooled_deployment_ids(
@ -15945,7 +15944,7 @@ class Router(DeprecatedRouterMembers):
)
if deployment is None:
exception: Final = await async_raise_no_deployment_exception(
litellm_router_instance=self,
ctx=self._core_ctx,
model=model,
parent_otel_span=parent_otel_span,
)
@ -16115,7 +16114,7 @@ class Router(DeprecatedRouterMembers):
if deployment is None:
exception: Final = await async_raise_no_deployment_exception(
litellm_router_instance=self,
ctx=self._core_ctx,
model=model,
parent_otel_span=parent_otel_span,
)
@ -16483,7 +16482,7 @@ class Router(DeprecatedRouterMembers):
if isinstance(selected_strategy.strategy, ComplexityRouter)
else (),
parent_model=model,
router=self,
ctx=self._core_ctx,
allow_escalation=isinstance(selected_strategy.strategy, ComplexityRouter)
and selected_strategy.strategy.config.enable_context_window_escalation,
messages=messages,
@ -16846,9 +16845,7 @@ class Router(DeprecatedRouterMembers):
parent_otel_span=parent_otel_span,
)
cooldown_deployments: Final = get_cooldown_deployments(
litellm_router_instance=self, parent_otel_span=parent_otel_span
)
cooldown_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
_pre_cooldown_deployments: Final = healthy_deployments
healthy_deployments = self._filter_cooldown_deployments(
healthy_deployments=healthy_deployments,
@ -16947,7 +16944,7 @@ class Router(DeprecatedRouterMembers):
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
raise RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
@ -16955,7 +16952,7 @@ class Router(DeprecatedRouterMembers):
cooldown_list=_cooldown_list,
model_ids=model_ids,
cooldown_deployments=get_cooldown_deployment_info(
self,
self._core_ctx,
self._deployment_ids(
self._region_routable_deployments(
request_deployments,
@ -16994,7 +16991,7 @@ class Router(DeprecatedRouterMembers):
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
raise RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
@ -17101,9 +17098,7 @@ class Router(DeprecatedRouterMembers):
healthy_deployments=pass_through_deployments,
parent_otel_span=parent_otel_span,
)
cooldown_deployments: Final = get_cooldown_deployments(
litellm_router_instance=self, parent_otel_span=parent_otel_span
)
cooldown_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
pass_through_deployments = self._filter_cooldown_deployments(
healthy_deployments=pass_through_deployments,
cooldown_deployments=cooldown_deployments,
@ -17132,7 +17127,7 @@ class Router(DeprecatedRouterMembers):
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
raise RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
@ -17140,7 +17135,7 @@ class Router(DeprecatedRouterMembers):
cooldown_list=_cooldown_list,
model_ids=pass_through_model_ids,
cooldown_deployments=get_cooldown_deployment_info(
self,
self._core_ctx,
self._deployment_ids(
self._region_routable_deployments(
request_deployments,
@ -17178,7 +17173,7 @@ class Router(DeprecatedRouterMembers):
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
raise RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,

View file

@ -33,19 +33,28 @@ as they were.
# repeat them unchanged, and typing the members themselves is the core's job.
import hashlib
import inspect
import json
import warnings
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, overload
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
if TYPE_CHECKING:
from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type
from opentelemetry.trace import Span
from litellm.caching.caching import DualCache, RedisCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.messages.dispatch import MessagesResult
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.router import Router
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
from litellm.router_strategy.auto_router.auto_router import AutoRouter
@ -53,12 +62,14 @@ if TYPE_CHECKING:
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.quality_router.quality_router import QualityRouter
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets
from litellm.router_utils.health_state_cache import DeploymentHealthCache
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.router_utils.ptu_shares import PTUTeamCeiling
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import (
AllowedFailsPolicy,
CustomRoutingStrategyBase,
Deployment,
DeploymentModelListingInfo,
DeploymentTypedDict,
@ -74,7 +85,7 @@ if TYPE_CHECKING:
SearchToolTypedDict,
TaggedPreRoutingStrategy,
)
from litellm.types.utils import BudgetConfig
from litellm.types.utils import BudgetConfig, EmbeddingResponse, ModelResponse
from litellm.types.utils import ModelInfo as ModelMapInfo
# The cores' untyped containers (bare ``list`` / ``dict``, element types Unknown), spelled with Any:
@ -82,6 +93,9 @@ if TYPE_CHECKING:
UntypedList: TypeAlias = list[Any] # pyright: ignore[reportExplicitAny] # see above
UntypedDict: TypeAlias = dict[Any, Any] # pyright: ignore[reportExplicitAny] # see above
# Re-exported for litellm.types, which may not import router_utils.
AttemptedFallbackTargets: TypeAlias = _AttemptedFallbackTargets
# What registry().plugins(kind) returns, one member per kind.
RegistryPlugins: TypeAlias = (
dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]]
@ -184,6 +198,73 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
return hashlib.sha256("".join(parts).encode()).hexdigest()
def router_init_args() -> tuple[str, ...]:
"""The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
from litellm.router import Router
return tuple(Router.get_valid_args())
def mark_advisor_orchestration_failure(exception: BaseException) -> None:
"""Tag an advisor orchestration failure, so the router does not cool down the deployment it picked."""
from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure as mark
mark(exception)
def fallback_chain(
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: the fallbacks config shape
model_group: str,
) -> tuple[list[str] | None, int | None]: # mutable-ok: get_fallback_model_group's type, unchanged
"""
The fallbacks configured for a model group, and the index of the generic ``*`` entry (was
``get_fallback_model_group``). Needs no router instance.
"""
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
return get_fallback_model_group(fallbacks=fallbacks, model_group=model_group)
def request_fallback_chain(
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: the fallbacks config shape
request_kwargs: Mapping[str, object],
model_group: str | None,
) -> tuple[list[str] | None, int | None]: # mutable-ok: get_fallback_model_group's type, unchanged
"""
``fallback_chain`` for a request: the group a pre-routing hook picked first, then the routed,
requested and original groups (was ``get_fallback_model_group_for_lookup_groups``).
"""
from litellm.router_utils.fallback_event_handlers import (
fallback_lookup_groups,
get_fallback_model_group_for_lookup_groups,
)
return get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs=request_kwargs, model_group=model_group)
)
def fallbacks_disabled(request_kwargs: Mapping[str, object]) -> bool:
"""Whether the request opted out of fallbacks (was ``fallbacks_disabled_for_request``)."""
from litellm.router_utils.fallback_event_handlers import fallbacks_disabled_for_request
return fallbacks_disabled_for_request(kwargs=request_kwargs)
def references_provider_scoped_resource(request_kwargs: Mapping[str, object]) -> bool:
"""Whether the request names a file, batch or fine-tuning job that only its own provider knows."""
from litellm.router_utils.fallback_event_handlers import PROVIDER_SCOPED_RESOURCE_KEYS
return any(request_kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS)
def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
"""The metadata bucket the router stamped the picked deployment into (was ``find_deployment_metadata``)."""
from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
return find_deployment_metadata(request_kwargs)
def _json_default_stable_id(obj: object) -> str:
from litellm.router import Router
@ -550,7 +631,8 @@ class RouterRegistry:
- ``group_set``: the set of configured group names (was the ``model_names`` attribute)
- ``ids``: deployment ids, of one group or all (was ``get_model_ids``)
- ``deployment_names``: (was the ``deployment_names`` attribute)
- ``team_public``: team public model names (was ``team_public_model_names``)
- ``team_public``: team public model names (was ``team_public_model_names``), or only the
ones ``team_id`` serves its own deployments under (was ``team_public_model_names_of``)
- ``litellm_models``: upstream model names a group resolves to (was ``resolved_litellm_models``)
- ``litellm_base_models``: the same, with each deployment's ``base_model`` where it sets one
(was ``resolved_litellm_models(..., use_base_model=True)``)
@ -568,7 +650,7 @@ class RouterRegistry:
if kind == "deployment_names":
return r._core_deployment_names
if kind == "team_public":
return r._core_team_public_model_names
return r._core_team_public_model_names if team_id is None else r.team_public_model_names_of(team_id)
if model_name is None:
raise TypeError(f"registry().names(kind={kind!r}) needs a model_name")
if kind == "litellm_models":
@ -832,6 +914,14 @@ class RouterRegistry:
"""Routing groups by name (was the ``routing_groups`` attribute)."""
return self._router._core_routing_groups
def routing_group(self, name: str) -> "RoutingGroup | None":
"""The routing group callable as ``name``, or None (was ``get_routing_group``)."""
return self._router.get_routing_group(name)
def client(self, deployment: Mapping[str, object], *, client_type: str | None = None) -> object:
"""The SDK client the router cached for a deployment, or None (was ``_get_client``)."""
return self._router._get_client(deployment=deployment, kwargs={}, client_type=client_type)
def access_groups(
self,
model_name: str | None = None,
@ -1128,6 +1218,41 @@ class RouterHealth:
"""The request priority and the kwargs without it (was ``split_request_priority``)."""
return self._router._core_split_request_priority(model, kwargs)
def failures_this_minute(self, deployment_id: str) -> int:
"""Failures counted against a deployment this minute (was ``get_deployment_failures_for_current_minute``)."""
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
get_deployment_failures_for_current_minute,
)
return get_deployment_failures_for_current_minute(self._router._core_ctx, deployment_id)
def record_failure(self, deployment_id: str, exception: Exception, exception_status: str | int) -> bool:
"""
Count a failure against a deployment this minute and cool it down once it is over its allowed
fails (was ``increment_deployment_failures_for_current_minute`` then ``set_cooldown_deployments``).
Returns whether the deployment was cooled down.
"""
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
increment_deployment_failures_for_current_minute,
)
ctx: Final = self._router._core_ctx
increment_deployment_failures_for_current_minute(ctx, deployment_id)
return set_cooldown_deployments(
ctx,
original_exception=exception,
exception_status=exception_status,
deployment=deployment_id,
time_to_cooldown=self._router._core_cooldown_time,
)
def priority_header_fields(
self, model: str, kwargs: Mapping[str, object], header: str | None
) -> Mapping[str, int | str]:
"""The request fields a priority header sets (was ``priority_header_update``)."""
return self._router.priority_header_update(model=model, kwargs=kwargs, header=header)
@dataclass
class ChangeSet:
@ -1202,6 +1327,40 @@ async def aapply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
return result
def _takes_legacy_llm_router(check: object) -> bool:
"""Whether a fallback check still declares ``llm_router`` instead of ``ctx`` (the pre-RoutingContext shape)."""
try:
params: Final = inspect.signature(check).parameters # pyright: ignore[reportArgumentType] # any callable
except (TypeError, ValueError):
return False
return "ctx" not in params and "llm_router" in params
async def _run_fallback_check(
check: "FallbackAccessCheck | FallbackBudgetCheck",
ctx: "RoutingContext",
*,
model: str,
request_kwargs: Mapping[str, object],
) -> bool:
if _takes_legacy_llm_router(check):
warnings.warn(
f"{type(check).__name__} takes llm_router=; fallback checks now receive ctx: RoutingContext. "
"Passing llm_router is deprecated and will be removed",
DeprecationWarning,
stacklevel=2,
)
legacy: Final = check # pyright: ignore[reportAny] # the old protocol shape, checked above
return await legacy(model=model, request_kwargs=request_kwargs, llm_router=ctx._router) # pyright: ignore[reportCallIssue, reportPrivateUsage] # see above
return await check(model=model, request_kwargs=request_kwargs, ctx=ctx)
class RegistryReader(Protocol):
"""Anything that answers ``registry()``: a Router or a RoutingContext. What proxy auth helpers read."""
def registry(self) -> "RouterRegistry": ...
class RoutingContext:
"""
What a routing plugin (strategy, pre-call check, sub-router) may use of the router. Plugins get
@ -1225,6 +1384,98 @@ class RoutingContext:
def health(self) -> RouterHealth:
return self._router.health()
def observers(self) -> RoutingObservers:
return self._router.observe()
def belongs_to(self, router: "Router") -> bool:
return self._router is router
@property
def explicit_allowed_fails(self) -> int | None:
"""``allowed_fails`` as the caller passed it, ``None`` when it came from the global default."""
return self._router.explicit_allowed_fails
async def select(
self,
model: str,
request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict
*,
kind: SelectKind = "pick",
messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list
input: str | list[object] | None = None, # mutable-ok: the selection cores take a list
specific_deployment: bool | None = False,
parent_otel_span: "Span | None" = None,
health_check_probe: bool = False,
) -> object:
return await self._router.select( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportCallIssue, reportArgumentType] # the selection cores return untyped deployment dicts
model,
request_kwargs,
kind=kind, # pyright: ignore[reportArgumentType] # forwarded as given, the overloads narrow it
messages=messages,
input=input,
specific_deployment=specific_deployment,
parent_otel_span=parent_otel_span,
health_check_probe=health_check_probe,
)
async def with_fallbacks(
self,
*args: object,
**kwargs: object, # kwargs-ok: the router's fallback loop takes the request's own keyword payload
) -> object:
"""Run a call through the router's retry and fallback loop (was ``async_function_with_fallbacks``)."""
return await self._router.async_function_with_fallbacks(*args, **kwargs)
async def acompletion(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "ModelResponse | CustomStreamWrapper":
return await self._router.acompletion(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
async def aresponses(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "ResponsesAPIResponse | BaseResponsesAPIStreamingIterator":
return await self._router.aresponses(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.aresponses returns
async def aembedding(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "EmbeddingResponse":
return await self._router.aembedding(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
def embedding(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "EmbeddingResponse":
return self._router.embedding(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
async def adecisions(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "OpenAIDecisionResponse | DecisionsResponse":
return await self._router.adecisions(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.adecisions returns
async def aanthropic_messages(
self,
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
) -> "MessagesResult":
return await self._router.aanthropic_messages(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.anthropic_messages returns
def apply(self, changes: ChangeSet) -> ApplyResult:
return self._router.apply(changes)
def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None:
self._router.register_plugin(plugin)
async def fallback_access_allowed(self, *, model: str, request_kwargs: Mapping[str, object]) -> bool:
check: Final = self._router.settings().fallback_access_check
return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs)
async def fallback_budget_allowed(self, *, model: str, request_kwargs: Mapping[str, object]) -> bool:
check: Final = self._router.settings().fallback_budget_check
return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs)
def pre_call_checks(
self,
model: str,
@ -1292,7 +1543,7 @@ class RoutingContext:
self,
kwargs: dict[str, object], # mutable-ok: Router._core_log_retry records the retry into the request kwargs
e: Exception,
) -> Mapping[str, object]:
) -> dict: # mutable-ok: the fallback loop edits it; Router._core_log_retry's type
return self._router._core_log_retry(kwargs=kwargs, e=e)

View file

@ -40,11 +40,11 @@ if TYPE_CHECKING:
from litellm.caching.caching import RedisCache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.router import Router
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
from litellm.router_utils.ptu_shares import PTUTeamCeiling
from litellm.types.router import (
CustomRoutingStrategyBase,
Deployment,
@ -838,6 +838,109 @@ class DeprecatedRouterMembers:
return _as_router(self)._core_log_retry(kwargs=kwargs, e=e)
# Router-family functions and classes that took the Router (``litellm_router_instance=``,
# ``router=``, ...) take its RoutingContext now. These keep the old spellings working.
_RoutingCallable: Final = TypeVar("_RoutingCallable", bound=Callable[..., object])
@functools.cache
def _router_class() -> "type[Router]":
from litellm.router import Router
return Router
def _note_renamed(label: str, old: str, instead: str, *, stacklevel: int = 3) -> None:
DEPRECATED_ROUTER_MEMBER_USES[label] += 1
warnings.warn(
f"{label}: {old} is deprecated; {instead} (the Router's RoutingContext) instead",
DeprecationWarning,
stacklevel=stacklevel, # the caller of the wrapper or descriptor that noted it
)
def _routing_context_of(value: object) -> object:
"""The Router's RoutingContext when ``value`` is a Router, else ``value`` as given."""
return value._core_ctx if isinstance(value, _router_class()) else value
def renamed_router_param(old: str, new: str = "ctx") -> Callable[[_RoutingCallable], _RoutingCallable]:
"""
Keep a router parameter that became a RoutingContext (``new``) callable the old way: ``old=``
still works, and so does a Router passed where ``new`` goes. Either gets the Router's
RoutingContext and a DeprecationWarning that names ``new``.
"""
def decorate(fn: _RoutingCallable) -> _RoutingCallable:
index: Final = list(inspect.signature(fn).parameters).index(new)
label: Final = f"{fn.__module__}.{fn.__qualname__}"
def convert(
args: tuple[object, ...],
kwargs: dict[str, object], # mutable-ok: the call's own kwargs
) -> tuple[object, ...]:
if old in kwargs:
if new in kwargs:
raise TypeError(f"{fn.__qualname__}() got both {old}= and {new}=")
_note_renamed(label, f"{old}=", f"pass {new}=", stacklevel=4)
kwargs[new] = _routing_context_of(kwargs.pop(old))
elif new in kwargs:
if isinstance(kwargs[new], _router_class()):
_note_renamed(label, f"a Router as {new}=", f"pass {new}=", stacklevel=4)
kwargs[new] = _routing_context_of(kwargs[new])
elif len(args) > index and isinstance(args[index], _router_class()):
_note_renamed(label, "passing a Router", f"pass {new}", stacklevel=4)
return (*args[:index], _routing_context_of(args[index]), *args[index + 1 :])
return args
# An async fn keeps an async wrapper, so iscoroutinefunction (and mock.patch's AsyncMock pick) still sees it
if inspect.iscoroutinefunction(fn):
@functools.wraps(fn)
async def acall(*args: object, **kwargs: object) -> object: # kwargs-ok: forwarded to fn as given
return await fn(*convert(args, kwargs), **kwargs) # pyright: ignore[reportAny] # fn's own result
return acall # pyright: ignore[reportReturnType] # same call shape as fn, see functools.wraps
@functools.wraps(fn)
def call(*args: object, **kwargs: object) -> object: # kwargs-ok: forwarded to fn as given
return fn(*convert(args, kwargs), **kwargs)
return call # pyright: ignore[reportReturnType] # same call shape as fn, see functools.wraps
return decorate
class RenamedRouterAttribute:
"""
A class attribute that keeps an old router attribute (``self.router``, ``self.litellm_router_instance``)
readable after it became the RoutingContext attribute ``new``: reading it returns that
RoutingContext, writing a Router stores the Router's RoutingContext, each with a DeprecationWarning.
"""
def __init__(self, new: str = "ctx") -> None:
self.new: Final = new
self.label = ""
def __set_name__(self, owner: type, name: str) -> None:
self.label = f"{owner.__module__}.{owner.__qualname__}.{name}"
@overload
def __get__(self, obj: None, owner: type | None = None) -> "RenamedRouterAttribute": ...
@overload
def __get__(self, obj: object, owner: type | None = None) -> object: ...
def __get__(self, obj: object | None, owner: type | None = None) -> object:
if obj is None:
return self
_note_renamed(self.label, "reading this attribute", f"read .{self.new}")
return getattr(obj, self.new) # pyright: ignore[reportAny] # the attribute named ``new`` holds the RoutingContext
def __set__(self, obj: object, value: object) -> None:
_note_renamed(self.label, "setting this attribute", f"set .{self.new}")
setattr(obj, self.new, _routing_context_of(value))
def _name_members_after_router() -> None:
# The old members were Router methods, so errors that name the method (a wrong-arity TypeError,
# or "litellm.router.Router.<name>() argument after ** must be a mapping") keep naming it the

View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
forwarded_internal_call_metadata,
parent_session_kwargs,
)
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
@ -29,11 +30,11 @@ if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
from semantic_router.routers.base import Route
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
from litellm.types.router import PreRoutingHookResponse
else:
Router = Any
RoutingContext = Any
PreRoutingHookResponse = Any
Route = Any
SemanticRouter = Any
@ -55,13 +56,15 @@ class _SemanticMatchOutcome(NamedTuple):
class AutoRouter(CustomLogger):
DEFAULT_AUTO_SYNC_VALUE = "local"
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
@renamed_router_param("litellm_router_instance")
def __init__(
self,
model_name: str,
default_model: str,
embedding_model: str,
litellm_router_instance: "Router",
ctx: "RoutingContext",
auto_router_config_path: str | None = None,
auto_router_config: str | None = None,
max_input_chars: int = DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
@ -75,7 +78,7 @@ class AutoRouter(CustomLogger):
auto_router_config: The config to use for the auto-router. You can either use this or auto_router_config_path, not both.
default_model: The default model to use if no route is found.
embedding_model: The embedding model to use for the auto-router.
litellm_router_instance: The instance of the LiteLLM Router.
ctx: The routing context of the owning router.
max_input_chars: Longest prompt, in characters, handed to the embedding model. Longer
prompts are cut to this length so an embedding context window far smaller than the
routed model's never fails the request.
@ -95,9 +98,9 @@ class AutoRouter(CustomLogger):
self.default_model = default_model
self.embedding_model: str = embedding_model
self.max_input_chars: int = max_input_chars
self.litellm_router_instance: Router = litellm_router_instance
self.ctx: Final = ctx
self.encoder: LiteLLMRouterEncoder = LiteLLMRouterEncoder(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
model_name=embedding_model,
max_input_chars=max_input_chars,
)

View file

@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, Final, Optional
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from pydantic import ConfigDict
from semantic_router.encoders import DenseEncoder
@ -6,11 +6,12 @@ from semantic_router.encoders.base import AsymmetricDenseMixin
import litellm
from litellm._logging import verbose_router_logger
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
else:
Router = Any
RoutingContext = Any
def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
@ -19,21 +20,22 @@ def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
:param embeds: The LiteLLM embedding response.
:return: A list of embeddings.
"""
if not embeds or not isinstance(embeds, litellm.EmbeddingResponse) or not embeds.data:
if not embeds or not embeds.data:
raise ValueError("No embeddings found in LiteLLM embedding response.")
return [x["embedding"] for x in embeds.data]
class CustomDenseEncoder(DenseEncoder):
model_config = ConfigDict(extra="allow")
litellm_router_instance: ClassVar = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
def __init__(self, litellm_router_instance: Optional["Router"] = None, **kwargs):
# Extract litellm_router_instance from kwargs if passed there
if "litellm_router_instance" in kwargs:
litellm_router_instance = kwargs.pop("litellm_router_instance")
@renamed_router_param("litellm_router_instance")
def __init__(self, ctx: Optional["RoutingContext"] = None, **kwargs):
if "ctx" in kwargs:
ctx = kwargs.pop("ctx")
super().__init__(**kwargs)
self.litellm_router_instance = litellm_router_instance
self.ctx = ctx
class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
@ -46,17 +48,18 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
type: str = "internal_litellm_router"
@renamed_router_param("litellm_router_instance")
def __init__(
self,
litellm_router_instance: "Router",
ctx: "RoutingContext",
model_name: str,
score_threshold: float | None = None,
max_input_chars: int = 0,
):
"""Initialize the LiteLLMEncoder.
:param litellm_router_instance: The instance of the LiteLLM Router.
:type litellm_router_instance: Router
:param ctx: The routing context of the owning router.
:type ctx: RoutingContext
:param model_name: The name of the embedding model to use. Must use LiteLLM naming
convention (e.g. "openai/text-embedding-3-small" or "mistral/mistral-embed").
:type model_name: str
@ -75,7 +78,7 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
score_threshold=score_threshold if score_threshold is not None else 0.3,
)
self.model_name = model_name
self.litellm_router_instance = litellm_router_instance
self.ctx = ctx
self.max_input_chars = max_input_chars
def _clamp(self, docs: list[str]) -> list[str]: # mutable-ok: embedding() takes `input: str | list`
@ -109,45 +112,37 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
return await self.aencode_queries(docs, **kwargs)
def encode_queries(self, docs: list[str], **kwargs) -> list[list[float]]:
if self.litellm_router_instance is None:
raise ValueError("litellm_router_instance is not set")
if self.ctx is None:
raise ValueError("ctx is not set")
try:
embeds: Final = self.litellm_router_instance.embedding(
input=self._clamp(docs), model=self.model_name, **kwargs
)
embeds: Final = self.ctx.embedding(input=self._clamp(docs), model=self.model_name, **kwargs)
return litellm_to_list(embeds)
except Exception as e:
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
def encode_documents(self, docs: list[str], **kwargs) -> list[list[float]]:
if self.litellm_router_instance is None:
raise ValueError("litellm_router_instance is not set")
if self.ctx is None:
raise ValueError("ctx is not set")
try:
embeds: Final = self.litellm_router_instance.embedding(
input=self._clamp(docs), model=self.model_name, **kwargs
)
embeds: Final = self.ctx.embedding(input=self._clamp(docs), model=self.model_name, **kwargs)
return litellm_to_list(embeds)
except Exception as e:
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
async def aencode_queries(self, docs: list[str], **kwargs) -> list[list[float]]:
if self.litellm_router_instance is None:
raise ValueError("litellm_router_instance is not set")
if self.ctx is None:
raise ValueError("ctx is not set")
try:
embeds: Final = await self.litellm_router_instance.aembedding(
input=self._clamp(docs), model=self.model_name, **kwargs
)
embeds: Final = await self.ctx.aembedding(input=self._clamp(docs), model=self.model_name, **kwargs)
return litellm_to_list(embeds)
except Exception as e:
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
async def aencode_documents(self, docs: list[str], **kwargs) -> list[list[float]]:
if self.litellm_router_instance is None:
raise ValueError("litellm_router_instance is not set")
if self.ctx is None:
raise ValueError("ctx is not set")
try:
embeds: Final = await self.litellm_router_instance.aembedding(
input=self._clamp(docs), model=self.model_name, **kwargs
)
embeds: Final = await self.ctx.aembedding(input=self._clamp(docs), model=self.model_name, **kwargs)
return litellm_to_list(embeds)
except Exception as e:
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e

View file

@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
from litellm.router_strategy.complexity_router.tier_predictor import (
@ -140,12 +141,12 @@ from .stall_detector import detect_stalled_task
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
from litellm.router_strategy.savings_baseline import Baseline
from litellm.types.router import PreRoutingHookResponse
else:
Router = Any
RoutingContext = Any
PreRoutingHookResponse = Any
SemanticRouter = Any
@ -1416,9 +1417,11 @@ class ComplexityRouter(CustomLogger):
- Question complexity (multiple questions)
"""
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
def _build_jev_client(self, config: OpenSourceClassifierConfig) -> JevClassifierClient | _JevConnectionError:
if config.deployment_name is not None:
return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name)
return RouterJevClassifierClient(self.ctx, config.deployment_name)
if config.provider in ("laya", "bespoke"):
from litellm.llms.oss_decision import oss_connection
@ -1459,10 +1462,11 @@ class ComplexityRouter(CustomLogger):
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
)
@renamed_router_param("litellm_router_instance")
def __init__(
self,
model_name: str,
litellm_router_instance: Router,
ctx: RoutingContext,
complexity_router_config: Mapping[str, object] | None = None,
default_model: str | None = None,
derive_savings_baseline: bool = True,
@ -1473,7 +1477,7 @@ class ComplexityRouter(CustomLogger):
Args:
model_name: The name of the model/deployment using this router.
litellm_router_instance: The LiteLLM Router instance.
ctx: The routing context of the owning router.
complexity_router_config: Optional configuration dict from proxy config.
default_model: Optional default model to use if tier cannot be determined.
derive_savings_baseline: False for callers whose decisions are never spend
@ -1481,7 +1485,7 @@ class ComplexityRouter(CustomLogger):
would leak deployment mappings the caller was not authorized for.
"""
self.model_name = model_name
self.litellm_router_instance = litellm_router_instance
self.ctx: Final = ctx
self._derive_savings_baseline = derive_savings_baseline
# Parse config - always create a new instance to avoid singleton mutation
@ -1698,7 +1702,7 @@ class ComplexityRouter(CustomLogger):
if not self._derive_savings_baseline:
return None
if not self._savings_baseline_derived:
self._savings_baseline = resolve_baseline(self.litellm_router_instance, self._hardest_tier_models())
self._savings_baseline = resolve_baseline(self.ctx, self._hardest_tier_models())
self._savings_baseline_derived = True
return self._savings_baseline
@ -2406,7 +2410,7 @@ class ComplexityRouter(CustomLogger):
)
budgeted: Final = await offload_token_count(fit_jev_request)(
build_request, prompt, system_prompt, jev_input_targets(self.litellm_router_instance, config), _truncate
build_request, prompt, system_prompt, jev_input_targets(self.ctx, config), _truncate
)
request: Final = budgeted.request
if request is None:
@ -2946,10 +2950,8 @@ class ComplexityRouter(CustomLogger):
"originating_request_masked": masked_originating_request(request_kwargs),
"body": {"model": llm_config.model, **payload},
}
classify: Final[Callable[..., Awaitable[ModelResponse | ResponsesAPIResponse]]] = (
self.litellm_router_instance.aresponses
if encrypted_task is not None
else self.litellm_router_instance.acompletion
classify: Final[Callable[..., Awaitable[ModelResponse | ResponsesAPIResponse]]] = ( # pyright: ignore[reportAssignmentType] # the context forwards the router endpoints untyped, non-streaming calls return these
self.ctx.aresponses if encrypted_task is not None else self.ctx.acompletion
)
classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000
@ -3148,7 +3150,7 @@ class ComplexityRouter(CustomLogger):
+ hashlib.sha256(json.dumps(cache_identity).encode()).hexdigest()
)
winner: Final = await claim_affinity_pin(
self.litellm_router_instance.cache,
self.ctx.health().store,
cache_key,
MappingProxyType({"model": model}),
self.config.session_affinity_ttl_seconds,
@ -3241,7 +3243,7 @@ class ComplexityRouter(CustomLogger):
model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {}
model_to_cost: Final[dict[str, float]] = {}
for name in available_models:
deployments = self.litellm_router_instance.deployments_named(name)
deployments = self.ctx.registry().deployments(name, source="named")
if not deployments:
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
model_to_cost[name] = 0.0
@ -3521,7 +3523,7 @@ class ComplexityRouter(CustomLogger):
):
return None
try:
model_info: Final = self.litellm_router_instance.get_router_model_info(
model_info: Final = self.ctx.registry().router_model_info(
deployment=cast(dict, deployment), # cast-ok: router deployments are plain dicts
received_model_name=group,
)
@ -3531,8 +3533,7 @@ class ComplexityRouter(CustomLogger):
return limit if isinstance(limit, int) else None
def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
deployments: Final = list_models(model_name=group) if callable(list_models) else None
deployments: Final = self.ctx.registry().deployments(model_name=group)
return tuple(deployments) if isinstance(deployments, list) else ()
def _group_output_ceiling(self, group: str) -> int | None:
@ -3803,7 +3804,7 @@ class ComplexityRouter(CustomLogger):
return declared is True
return model_verdict((deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name)
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
deployments: Final = self.ctx.registry().deployments(model_name=model_name)
if not deployments:
return (model_verdict(model_name),)
return tuple(deployment_verdict(deployment) for deployment in deployments)
@ -4000,9 +4001,10 @@ class ComplexityRouter(CustomLogger):
probe_kwargs: Final = dict(request_kwargs)
try:
deployments: Final = await self.litellm_router_instance.async_get_healthy_deployments(
model=model_name,
request_kwargs=probe_kwargs,
deployments: Final = await self.ctx.select(
model_name,
probe_kwargs,
kind="healthy",
messages=messages,
input=input,
parent_otel_span=get_parent_otel_span_from_kwargs(request_kwargs),
@ -4241,7 +4243,7 @@ class ComplexityRouter(CustomLogger):
routelayer: Final = SemanticRouter(
routes=routes,
encoder=LiteLLMRouterEncoder(
litellm_router_instance=self.litellm_router_instance,
ctx=self.ctx,
model_name=embedding_model,
score_threshold=self.config.match_threshold,
),
@ -4535,7 +4537,7 @@ class ComplexityRouter(CustomLogger):
if cache_key is not None and pin_replay_allowed:
pinned_value: Final = await read_affinity_pin(
self.litellm_router_instance.cache, cache_key, self.config.session_affinity_ttl_seconds
self.ctx.health().store, cache_key, self.config.session_affinity_ttl_seconds
)
pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names())
if pinned_pin is not None:
@ -4634,7 +4636,7 @@ class ComplexityRouter(CustomLogger):
else floor_model
)
if escalated:
await self.litellm_router_instance.cache.async_set_cache(
await self.ctx.health().store.async_set_cache(
key=cache_key,
value=_session_affinity_cache_value(session_model, resolved_pin_tier),
ttl=self.config.session_affinity_ttl_seconds,
@ -4737,7 +4739,7 @@ class ComplexityRouter(CustomLogger):
and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None
)
if pinnable and cache_key is not None and response is not None:
await self.litellm_router_instance.cache.async_set_cache(
await self.ctx.health().store.async_set_cache(
key=cache_key,
value=_session_affinity_cache_value(
response.model,
@ -4759,7 +4761,7 @@ class ComplexityRouter(CustomLogger):
from litellm.proxy.common_utils.cache_aware_routing import choose_cached_model
choice: Final = await choose_cached_model(
router=self.litellm_router_instance,
ctx=self.ctx,
config=self.config,
params_for_model=partial(self._litellm_params_for_model, request_kwargs=request_kwargs),
response=response,

View file

@ -27,10 +27,11 @@ from litellm.llms.compaction import (
dispatch,
get_native_compaction_provider,
)
from litellm.router_deprecated import renamed_router_param
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
Surface: TypeAlias = Literal["chat", "messages", "responses"]
_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType(
@ -96,13 +97,14 @@ def initialize_compaction_state(kwargs: Mapping[str, object], surface: Surface)
return existing if isinstance(existing, CompactionState) else CompactionState(surface=surface)
@renamed_router_param("router")
async def arm_compaction(
kwargs: Mapping[str, object],
config: ContextCompactionConfig | Literal[False] | None,
candidates: tuple[str, ...] = (),
parent_model: str | None = None,
*,
router: Router | None = None,
ctx: RoutingContext | None = None,
allow_escalation: bool = False,
messages: Sequence[Mapping[str, object]] | None = None,
) -> None:
@ -111,7 +113,7 @@ async def arm_compaction(
state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None
state.candidates = candidates
state.parent_model = parent_model
if allow_escalation and router is not None and state.config is not None:
if allow_escalation and ctx is not None and state.config is not None:
payload: Final = MappingProxyType(
{
**kwargs,
@ -119,11 +121,11 @@ async def arm_compaction(
**({"messages": messages} if messages is not None and state.surface != "responses" else {}),
}
)
if not await _has_compactor(router, state, payload):
if not await _has_compactor(ctx, state, payload):
state.config = None
async def _has_compactor(router: Router, state: CompactionState, payload: Mapping[str, object]) -> bool:
async def _has_compactor(ctx: RoutingContext, state: CompactionState, payload: Mapping[str, object]) -> bool:
from litellm.exceptions import ContextWindowExceededError
if state.surface is None or state.config is None:
@ -131,7 +133,7 @@ async def _has_compactor(router: Router, state: CompactionState, payload: Mappin
try:
instructions, prefix, _ = _portable_history(payload, state.surface)
await _compactor_model(
router, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens)
ctx, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens)
)
return True
except ContextWindowExceededError:
@ -237,9 +239,9 @@ def _history(
return instructions, prefix, conversation[split:]
async def _count(router: Router, payload: Mapping[str, object]) -> int:
async def _count(ctx: RoutingContext, payload: Mapping[str, object]) -> int:
return await asyncio.to_thread(
router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter
ctx.count_pre_call_check_tokens,
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None,
input=_INPUT.validate_python(payload.get("input")),
request_kwargs=payload,
@ -247,11 +249,11 @@ async def _count(router: Router, payload: Mapping[str, object]) -> int:
def _budget(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
) -> InputBudget:
model: Final = str(payload.get("model", ""))
info: Final = _MAPPING.validate_python(
router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
ctx.registry().router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
)
raw_window: Final = info.get("max_input_tokens")
window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None
@ -269,9 +271,9 @@ def _budget(
async def _compactor_model(
router: Router, state: CompactionState, payload: Mapping[str, object]
ctx: RoutingContext, state: CompactionState, payload: Mapping[str, object]
) -> tuple[str, NativeCompactionProvider]:
needed: Final = await _count(router, payload)
needed: Final = await _count(ctx, payload)
candidates: Final = (
(state.config.model,) if state.config is not None and state.config.model is not None else state.candidates
)
@ -279,13 +281,13 @@ async def _compactor_model(
(
(candidate, provider)
for candidate in candidates
if (deployments := tuple(router.get_model_list(model_name=candidate) or ()))
if (deployments := tuple(ctx.registry().deployments(model_name=candidate) or ()))
and (provider := get_native_compaction_provider(_MAPPING.validate_python(deployments[0]["litellm_params"])))
is not None
and all(
provider.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"]))
and provider.compatible_defaults(params)
and (budget := _budget(router, deployment, payload, 0.9)).available is not None
and (budget := _budget(ctx, deployment, payload, 0.9)).available is not None
and needed <= budget.available
for deployment in deployments
)
@ -320,7 +322,7 @@ def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[s
async def _generate_summary(
router: Router,
ctx: RoutingContext,
provider: NativeCompactionProvider,
protocol: CompactionProtocol,
payload: Mapping[str, object],
@ -330,7 +332,7 @@ async def _generate_summary(
executor: Final = compaction_executor.get()
with native_compaction_call():
response: Final = await asyncio.wait_for(
executor(protocol, payload, parent_model) if executor is not None else dispatch(router, protocol, payload),
executor(protocol, payload, parent_model) if executor is not None else dispatch(ctx, protocol, payload),
timeout=timeout,
)
summary: Final = provider.extract_summary(protocol, response)
@ -362,32 +364,33 @@ def _compactor_input(
)
@renamed_router_param("router")
async def compact_to_fit(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
) -> Mapping[str, object]:
from litellm.exceptions import ContextWindowExceededError
try:
return await _compact_to_fit(router, deployment, payload, surface)
return await _compact_to_fit(ctx, deployment, payload, surface)
except ContextWindowExceededError:
window: Final = _budget(router, deployment, payload, 1.0).window
if not _native_child.get() and window is not None and await _count(router, payload) <= window:
window: Final = _budget(ctx, deployment, payload, 1.0).window
if not _native_child.get() and window is not None and await _count(ctx, payload) <= window:
return payload
raise
async def _check_client_managed_admission(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object]
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object]
) -> None:
if not router.enable_pre_call_checks:
if not ctx.settings().enable_pre_call_checks:
return
router._pre_call_checks( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # restore legacy admission after deployment defaults
ctx.pre_call_checks(
model=str(payload["model"]),
healthy_deployments=_ITEMS.validate_python((deployment,)),
healthy_deployments=list[object](_ITEMS.validate_python((deployment,))),
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, # pyright: ignore[reportArgumentType] # legacy annotation omits structured content
input=_INPUT.validate_python(payload.get("input")),
request_kwargs=_DICT.validate_python(payload),
input_token_count=await _count(router, payload),
input_token_count=await _count(ctx, payload),
skip_inline_token_count=True,
)
@ -419,22 +422,22 @@ def _portable_history(
async def _compact_to_fit(
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
) -> Mapping[str, object]:
state: Final = payload.get(_STATE_KEY)
config: Final = state.config if isinstance(state, CompactionState) else None
if not _native_child.get() and (config is None or _client_managed(payload)):
if config is not None:
await _check_client_managed_admission(router, deployment, payload)
await _check_client_managed_admission(ctx, deployment, payload)
return payload
model: Final = str(payload["model"])
limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9)
limits: Final = _budget(ctx, deployment, payload, config.trigger_ratio if config is not None else 0.9)
budget: Final = limits.available
if budget is None or budget <= 0:
if (
config is not None
and config.model is None
and (limits.window is None or await _count(router, payload) <= limits.window)
and (limits.window is None or await _count(ctx, payload) <= limits.window)
):
return payload
_unavailable(model, "A known input window and a smaller output allowance are required")
@ -443,18 +446,18 @@ async def _compact_to_fit(
if (
child_provider is None
or not child_provider.compatible_defaults(payload)
or await _count(router, payload) > budget
or await _count(ctx, payload) > budget
):
_reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget")
return payload
if await _count(router, payload) <= budget:
if await _count(ctx, payload) <= budget:
return payload
if surface is None or config is None or not isinstance(state, CompactionState):
_unavailable(model, "This request surface cannot be compacted")
key: Final = "input" if surface == "responses" else "messages"
instructions, prefix, tail = _portable_history(payload, surface)
retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))})
if await _count(router, retained) >= budget:
if await _count(ctx, retained) >= budget:
_unavailable(model, "Retained instructions, tools and the latest turn leave no room for a summary")
older: Final = _compactor_input(payload, surface, instructions, prefix, config.max_tokens)
metadata: Final = sanitized_forwardable_call_metadata(
@ -481,7 +484,7 @@ async def _compact_to_fit(
**parent_session_kwargs(payload),
}
)
compactor, provider = await _compactor_model(router, state, request)
compactor, provider = await _compactor_model(ctx, state, request)
child: Final = MappingProxyType({**request, **provider.request_kwargs(), "model": compactor})
identity: Final = hashlib.sha256(
json.dumps(
@ -505,7 +508,7 @@ async def _compact_to_fit(
state.summary = (
identity,
asyncio.create_task(
_generate_summary(router, provider, protocol, child, config.timeout_seconds, state.parent_model)
_generate_summary(ctx, provider, protocol, child, config.timeout_seconds, state.parent_model)
),
)
if state.summary[0] != identity:
@ -515,6 +518,6 @@ async def _compact_to_fit(
{"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary}
)
compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))})
if await _count(router, compacted) > budget:
if await _count(ctx, compacted) > budget:
_unavailable(model, "The summary and retained conversation still exceed the selected deployment's budget")
return compacted

View file

@ -239,10 +239,10 @@ def run_eval() -> tuple[int, int, list[dict]]:
Tuple of (passed, total, failures)
"""
# Create router with default config
mock_router: Final = MagicMock()
mock_ctx: Final = MagicMock()
router: Final = ComplexityRouter(
model_name="eval-router",
litellm_router_instance=mock_router,
ctx=mock_ctx,
)
passed = 0

View file

@ -23,13 +23,13 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG, is_ai_decide_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler
from litellm.router_deprecated import renamed_router_param
from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS
from litellm.types.decisions import DecisionsResponse
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, StandardLoggingRoutingDecision
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.complexity_router.config import OpenSourceClassifierConfig
from litellm.types.router import Deployment
@ -68,14 +68,12 @@ class JevBudgetedRequest:
telemetry: StandardLoggingRoutingDecision
def jev_input_targets(router: "Router", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
from litellm.router_api import RoutingContext
def jev_input_targets(context: "RoutingContext", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
from litellm.types.router import Deployment
if config.deployment_name is None:
provider: Final = "typesafe" if config.provider == "jev" else config.provider
return (_jev_input_target({"model": f"{provider}/{config.model}"}, None),)
context: Final = RoutingContext(router)
deployments: Final = context.registry().deployments(model_name=config.deployment_name) or ()
return tuple(
_jev_deployment_input_target(context, Deployment.model_validate(deployment)) for deployment in deployments
@ -240,30 +238,13 @@ class JevClassifierClient(Protocol):
) -> JevSystemOneResponse: ...
class _DecisionsRouter(Protocol):
async def adecisions(
self,
*,
model: str,
state: str,
questions: Mapping[str, object],
timeout: float,
num_retries: int,
disable_fallbacks: bool,
metadata: Mapping[str, object],
proxy_server_request: Mapping[str, object],
turn_off_message_logging: bool | None,
litellm_session_id: str = "",
litellm_trace_id: str = "",
) -> object: ...
_PARENT_METADATA: Final = TypeAdapter[Mapping[str, object] | None](Mapping[str, object] | None)
class RouterJevClassifierClient:
def __init__(self, router: "Router", deployment_name: str) -> None:
self._router: _DecisionsRouter = router
@renamed_router_param("router")
def __init__(self, ctx: "RoutingContext", deployment_name: str) -> None:
self.ctx: Final = ctx
self._deployment_name = deployment_name
async def evaluate(
@ -285,7 +266,7 @@ class RouterJevClassifierClient:
"questions": {key: value.model_dump() for key, value in request.questions.items()},
}
response: Final = DecisionsResponse.model_validate(
await self._router.adecisions(
await self.ctx.adecisions(
model=self._deployment_name,
state=request.state,
questions={key: value.model_dump() for key, value in request.questions.items()},

View file

@ -16,10 +16,12 @@ from pydantic import ValidationError
from litellm._logging import verbose_router_logger
from litellm.router import CustomRoutingStrategyBase
from litellm.router_api import ChangeSet
from litellm.router_deprecated import renamed_router_param
from litellm.types.lar1 import LAR1Metadata, LAR1Time
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
DEFAULT_THRESHOLDS: Final[dict[str, float]] = {"low": 0.3, "medium": 0.5, "high": 0.7}
@ -41,16 +43,17 @@ def lar1_thresholds_from_args(
}
@renamed_router_param("router")
def apply_lar1_routing_strategy(
router: Router,
ctx: RoutingContext,
routing_strategy_args: Mapping[str, object] | None = None,
) -> None:
strategy: Final = LAR1RoutingStrategy(
router_instance=router,
ctx=ctx,
thresholds=lar1_thresholds_from_args(routing_strategy_args),
)
router.routing_strategy = "lar1"
router.set_custom_routing_strategy(strategy)
ctx.apply(ChangeSet(assign={"routing_strategy": "lar1"}))
ctx.register_plugin(strategy)
def _normalize_thresholds(thresholds: dict[str, float] | None) -> dict[str, float]:
@ -78,12 +81,13 @@ def _parse_lar1_metadata(request_kwargs: dict) -> LAR1Metadata:
class LAR1RoutingStrategy(CustomRoutingStrategyBase):
@renamed_router_param("router_instance")
def __init__(
self,
router_instance: Router | None = None,
ctx: RoutingContext | None = None,
thresholds: dict[str, float] | None = None,
):
self._router = router_instance
self._ctx = ctx
self.thresholds = _normalize_thresholds(thresholds)
async def async_get_available_deployment(
@ -96,7 +100,7 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
):
if request_kwargs is None:
request_kwargs = {}
if self._router is None:
if self._ctx is None:
return None
lar1: Final = _parse_lar1_metadata(request_kwargs)
@ -104,9 +108,10 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
evidence: Final = tuple(e.value for e in lar1.evidence)
time_dim: Final = lar1.time.value
healthy: Final = await self._router.async_get_healthy_deployments(
model=model,
request_kwargs=request_kwargs,
healthy: Final = await self._ctx.select(
model,
request_kwargs,
kind="healthy",
messages=messages,
input=input,
specific_deployment=specific_deployment,
@ -114,7 +119,7 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
if isinstance(healthy, dict):
return healthy
if not healthy:
if not isinstance(healthy, list) or not healthy:
return None
target: Final = self._classify_request(confidence, evidence, time_dim)

View file

@ -11,7 +11,7 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:

View file

@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
)
@ -29,10 +30,10 @@ from litellm.types.utils import StandardLoggingRoutingDecision
from .config import QualityRouterConfig, RoutingPreferences
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.types.router import PreRoutingHookResponse
else:
Router = Any
RoutingContext = Any
PreRoutingHookResponse = Any
@ -42,15 +43,18 @@ class QualityRouter(CustomLogger):
keyword override.
"""
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
@renamed_router_param("litellm_router_instance")
def __init__(
self,
model_name: str,
litellm_router_instance: "Router",
ctx: "RoutingContext",
default_model: str | None = None,
quality_router_config: Mapping[str, object] | None = None,
):
self.model_name = model_name
self.litellm_router_instance = litellm_router_instance
self.ctx: Final = ctx
if quality_router_config:
self.config = QualityRouterConfig.model_validate(dict(**quality_router_config))
@ -64,7 +68,7 @@ class QualityRouter(CustomLogger):
# Internal scorer — re-use the existing rule-based classifier.
self._scorer = ComplexityRouter(
model_name=f"{model_name}::scorer",
litellm_router_instance=litellm_router_instance,
ctx=ctx,
)
# Per-model indices populated alongside the tier index. `_model_keywords`
@ -158,7 +162,7 @@ class QualityRouter(CustomLogger):
`_model_quality`, and `_model_cost`. Raises if any listed model is
missing `litellm_routing_preferences`.
"""
model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or []
model_list: Final = self.ctx.registry().all() or []
available: Final = set(self.config.available_models)
# Track which available models we've matched so we can error on missing.

View file

@ -18,10 +18,11 @@ from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Final, NamedTuple
from litellm._logging import verbose_router_logger
from litellm.router_deprecated import renamed_router_param
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
_REFERENCE_REQUEST: Final = Usage(
@ -70,7 +71,7 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
return f"{provider}/{resolved}"
def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
"""The candidates a tier entry actually calls, each with its own pricing key.
`litellm_params.model` is not always a model: on Azure it is the deployment name,
@ -80,7 +81,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
straight at a provider model rather than at a configured group, and prices under
its own name because there is no deployment to override it.
"""
deployments: Final = router.deployments_named(group_name)
deployments: Final = ctx.registry().deployments(group_name, source="named")
if not deployments:
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
@ -100,7 +101,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None)
def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None:
def _priced(ctx: "RoutingContext", candidate: Baseline) -> tuple[float, Baseline] | None:
"""``(cost_of_the_reference_request, candidate)``, or ``None`` when unpriceable.
"Most expensive" is a property of a request, not of a rate: a deployment dearer per
@ -114,7 +115,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
provider, _, model_name = candidate.model.partition("/")
try:
info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model)
info: Final = ctx.registry().deployment_model_info(candidate.deployment_id or "", candidate.model, cached=True)
if info is None:
return None
prompt_cost, completion_cost = generic_cost_per_token(
@ -133,16 +134,17 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
return (cost, candidate)
def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None:
def _most_expensive(ctx: "RoutingContext", candidates: Iterable[Baseline]) -> Baseline | None:
"""The candidate that would have cost the most on the reference request."""
priced: Final = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None)
priced: Final = tuple(r for candidate in candidates if (r := _priced(ctx, candidate)) is not None)
if not priced:
verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled")
return None
return max(priced)[1]
def resolve_baseline(router: "Router", group_names: Iterable[str]) -> Baseline | None:
@renamed_router_param("router")
def resolve_baseline(ctx: "RoutingContext", group_names: Iterable[str]) -> Baseline | None:
"""The derived baseline for a router whose hardest tier offers ``group_names``.
Holds no cache of its own; each pricing pass walks the pool, so the caller is
@ -155,7 +157,7 @@ def resolve_baseline(router: "Router", group_names: Iterable[str]) -> Baseline |
over; an unresolvable baseline zeroes the savings driver instead.
"""
try:
return _most_expensive(router, (c for name in group_names for c in _models_in(router, name)))
return _most_expensive(ctx, (c for name in group_names for c in _models_in(ctx, name)))
except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric
verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e)
return None

View file

@ -12,7 +12,7 @@ import re
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypedDict, overload
from typing_extensions import ReadOnly
@ -23,14 +23,11 @@ from litellm.constants import (
ROUTING_REQUEST_TAGS_METADATA_KEY,
)
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
from litellm.router_deprecated import renamed_router_param
from litellm.types.router import BaggageMappedTagsStamp, ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
else:
LitellmRouter = Any
from litellm.router_api import RoutingContext
class _TagLitellmParamsLike(Protocol):
@ -394,18 +391,18 @@ def _resolve_constraint_only_pool(
def _all_deployments_or_fallback(
llm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model: str,
fallback: _DeploymentPool,
) -> Sequence[_DeploymentLike | DeploymentTypedDict] | Mapping[_DeploymentLike, object]:
try:
return llm_router_instance.get_all_deployments(model_name=model)
return ctx.registry().deployments(model, source="all")
except Exception: # noqa: BLE001 # fail safe toward today's healthy-only behavior on lookup errors
return fallback
def _chain_tag_filtering_override(
llm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model: str,
healthy_deployments: _DeploymentPool,
) -> bool | None:
@ -418,7 +415,7 @@ def _chain_tag_filtering_override(
# repeatedly failing that one deployment into cooldown. Falls back to
# healthy_deployments on a lookup error, preserving today's behavior rather
# than crashing the request.
all_deployments: Final = _all_deployments_or_fallback(llm_router_instance, model, healthy_deployments)
all_deployments: Final = _all_deployments_or_fallback(ctx, model, healthy_deployments)
for d in all_deployments:
value = (d.get("model_info") or MappingProxyType({})).get("enable_tag_filtering")
if value is not None:
@ -458,7 +455,7 @@ def _tags_that_route(request_tags: Sequence[str], metadata: object, routing_pref
def _tag_known_to_group(
llm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model: str,
positive_tags: Sequence[str],
routing_confirmed: frozenset[str],
@ -467,7 +464,7 @@ def _tag_known_to_group(
if tag_set & routing_confirmed:
return True
try:
all_deployments: Final = llm_router_instance.get_all_deployments(model_name=model)
all_deployments: Final = ctx.registry().deployments(model, source="all")
except Exception: # noqa: BLE001 # fail safe toward "unrecognized" so lookup errors preserve the existing silent-fallback behavior
return False
return any(
@ -557,8 +554,9 @@ def _tag_sequence(tags: object) -> Sequence[object]:
return typed_tags
@renamed_router_param("llm_router_instance")
async def get_deployments_for_tag(
llm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model: str, # used to raise the correct error
healthy_deployments: _DeploymentPool,
request_kwargs: _RequestKwargsLike | None = None,
@ -587,11 +585,9 @@ async def get_deployments_for_tag(
return healthy_deployments
request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering")
chain_enable_tag_filtering: Final = _chain_tag_filtering_override(llm_router_instance, model, healthy_deployments)
chain_enable_tag_filtering: Final = _chain_tag_filtering_override(ctx, model, healthy_deployments)
chain_default: Final = (
chain_enable_tag_filtering
if chain_enable_tag_filtering is not None
else llm_router_instance.enable_tag_filtering
chain_enable_tag_filtering if chain_enable_tag_filtering is not None else ctx.settings().enable_tag_filtering
)
if request_enable_tag_filtering is not True and chain_default is not True:
return healthy_deployments
@ -602,8 +598,8 @@ async def get_deployments_for_tag(
request_tags: Final = without_baggage_mapped_tags(
_request_tags_after_router_consumption(metadata, model), metadata
)
match_any: Final = llm_router_instance.tag_filtering_match_any
routing_prefix: Final = llm_router_instance.tag_routing_prefix or ""
match_any: Final = ctx.settings().tag_filtering_match_any
routing_prefix: Final = ctx.settings().tag_routing_prefix or ""
# Build header strings for regex matching from what the proxy already stores.
# Currently we match against User-Agent; format matches "^User-Agent: claude-code/..."
@ -703,7 +699,7 @@ async def get_deployments_for_tag(
if (
len(new_healthy_deployments) == 0
and positive_tags
and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed)
and _tag_known_to_group(ctx, model, positive_tags, routing_confirmed)
):
return _resolve_or_fail_open(
request_deployments,

View file

@ -1,16 +1,13 @@
from types import TracebackType
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType
from litellm.router_deprecated import renamed_router_param
from litellm.types.router import RouterErrors
from litellm.utils import calculate_max_parallel_requests
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
else:
LitellmRouter = Any
from litellm.router_api import RoutingContext
class MaxParallelRequestsLimit:
@ -52,7 +49,8 @@ class MaxParallelRequestsLimit:
class InitalizeCachedClient:
@staticmethod
def set_max_parallel_requests_client(litellm_router_instance: LitellmRouter, model: dict):
@renamed_router_param("litellm_router_instance")
def set_max_parallel_requests_client(ctx: "RoutingContext", model: dict):
litellm_params: Final = model.get("litellm_params", {})
model_id: Final = model["model_info"]["id"]
rpm: Final = litellm_params.get("rpm", None)
@ -62,7 +60,7 @@ class InitalizeCachedClient:
rpm=rpm,
max_parallel_requests=max_parallel_requests,
tpm=tpm,
default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests,
default_max_parallel_requests=ctx.settings().default_max_parallel_requests,
)
if calculated_max_parallel_requests:
limit: Final = MaxParallelRequestsLimit(
@ -71,7 +69,7 @@ class InitalizeCachedClient:
model_group=model.get("model_name", ""),
)
cache_key: Final = f"{model_id}_max_parallel_requests_client"
litellm_router_instance.cache.set_cache(
ctx.health().store.set_cache(
key=cache_key,
value=limit,
local_only=True,

View file

@ -7,19 +7,18 @@ from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm._logging import verbose_logger
from litellm.router_deprecated import renamed_router_param
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
from litellm.integrations.prometheus import PrometheusLogger
from litellm.router_api import RoutingContext
else:
LitellmRouter = Any
PrometheusLogger = Any
@renamed_router_param("litellm_router_instance")
async def router_cooldown_event_callback(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment_id: str,
exception_status: str | int,
cooldown_time: float | None,
@ -31,7 +30,7 @@ async def router_cooldown_event_callback(
- Increments cooldown metric for deployment on Prometheus
"""
verbose_logger.debug("In router_cooldown_event_callback - updating prometheus")
_deployment: Final = litellm_router_instance.get_deployment(model_id=deployment_id)
_deployment: Final = ctx.registry().deployment(deployment_id)
if _deployment is None:
verbose_logger.warning(
"in router_cooldown_event_callback but _deployment is None for deployment_id=%s. Doing nothing",

View file

@ -29,6 +29,7 @@ from litellm.constants import (
from litellm.exceptions import MidStreamFallbackError
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials
from litellm.router_deprecated import renamed_router_param
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
from litellm.types.router import Deployment
@ -48,12 +49,10 @@ from .router_callbacks.track_deployment_metrics import (
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.router import Router as _Router
from litellm.router_api import RoutingContext
LitellmRouter = _Router
Span = _Span
else:
LitellmRouter = Any
Span = Any
_ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
@ -137,7 +136,7 @@ def _first_present(*sources: Mapping[str, Any] | None, key: str) -> int | float
def _get_deployment_cooldown_policy(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str,
) -> tuple[Mapping[str, int] | None, int | None]:
"""Return (allowed_fails_policy, allowed_fails) from deployment model_info, or (None, None).
@ -149,7 +148,7 @@ def _get_deployment_cooldown_policy(
field placed there would leak into the outgoing LLM request instead of staying
router-internal.
"""
dep: Final = litellm_router_instance.get_model_info(id=deployment)
dep: Final = ctx.registry().deployment_dict(deployment)
if dep is None:
return None, None
mi: Final[Mapping[str, Any]] = dep.get("model_info") or MappingProxyType({})
@ -175,7 +174,7 @@ def _resolve_allowed_fails_from_policy(
def _should_cooldown_based_on_deployment_policy(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str,
original_exception: Exception,
dep_policy: Mapping[str, int] | None,
@ -199,10 +198,7 @@ def _should_cooldown_based_on_deployment_policy(
if (
allowed_fails_from_policy is None
and is_single_deployment_model_group
and (
dep_allowed_fails is not None
or litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None
)
and (dep_allowed_fails is not None or ctx.settings().allowed_fails_for(original_exception) is None)
):
return False
@ -215,7 +211,7 @@ def _should_cooldown_based_on_deployment_policy(
else ("generic" if dep_allowed_fails is not None else None)
)
dep: Final = litellm_router_instance.get_model_info(id=deployment)
dep: Final = ctx.registry().deployment_dict(deployment)
cooldown_time_override: Final = (
_first_present(dep.get("model_info"), dep.get("litellm_params"), key="cooldown_time")
if dep is not None
@ -223,7 +219,7 @@ def _should_cooldown_based_on_deployment_policy(
)
return should_cooldown_based_on_allowed_fails_policy(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment=deployment,
original_exception=original_exception,
allowed_fails_override=allowed_fails_override,
@ -233,7 +229,7 @@ def _should_cooldown_based_on_deployment_policy(
def _has_explicit_allowed_fails_policy_for_exception(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str | None,
original_exception: Exception,
) -> bool:
@ -254,12 +250,12 @@ def _has_explicit_allowed_fails_policy_for_exception(
"""
if deployment is None:
return False
dep_policy, _ = _get_deployment_cooldown_policy(litellm_router_instance, deployment)
dep_policy, _ = _get_deployment_cooldown_policy(ctx, deployment)
return _resolve_allowed_fails_from_policy(dep_policy, original_exception) is not None
def _is_cooldown_required(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model_id: str,
exception_status: str | int,
exception_str: str | None = None,
@ -320,7 +316,7 @@ def _is_per_user_provider_request(request_kwargs: Mapping[str, object]) -> bool:
def _should_run_cooldown_logic(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str | None,
exception_status: str | int,
original_exception: Exception,
@ -335,10 +331,10 @@ def _should_run_cooldown_logic(
- router.disable_cooldowns is True
- deployment is None
- _is_cooldown_required() returns False
- deployment is in litellm_router_instance.provider_default_deployment_ids
- deployment is in ctx.settings().provider_default_deployment_ids
- exception_status is not one that should be immediately retried (e.g. 401)
"""
if deployment is None or litellm_router_instance.get_model_group(id=deployment) is None:
if deployment is None or ctx.registry().resolve(deployment, kind="group_of_id") is None:
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: deployment id is none or model group can't be found."
)
@ -369,7 +365,7 @@ def _should_run_cooldown_logic(
return False
if deployment is not None:
failed_deployment: Final = litellm_router_instance.get_deployment(model_id=deployment)
failed_deployment: Final = ctx.registry().deployment(deployment)
if failed_deployment is not None:
from litellm.llms.github_copilot.per_user_auth import (
github_copilot_per_user_credential_name,
@ -389,7 +385,7 @@ def _should_run_cooldown_logic(
verbose_router_logger.debug("Should Not Run Cooldown Logic: time_to_cooldown is effectively 0")
return False
if litellm_router_instance.disable_cooldowns:
if ctx.settings().disable_cooldowns:
verbose_router_logger.debug("Should Not Run Cooldown Logic: disable_cooldowns is True")
return False
@ -398,19 +394,19 @@ def _should_run_cooldown_logic(
return False
if not _is_cooldown_required(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
model_id=deployment,
exception_status=exception_status,
exception_str=str(original_exception),
) and not _has_explicit_allowed_fails_policy_for_exception(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment=deployment,
original_exception=original_exception,
):
verbose_router_logger.debug("Should Not Run Cooldown Logic: _is_cooldown_required returned False")
return False
if deployment in litellm_router_instance.provider_default_deployment_ids:
if deployment in ctx.settings().provider_default_deployment_ids:
verbose_router_logger.debug("Should Not Run Cooldown Logic: deployment is in provider_default_deployment_ids")
return False
@ -448,10 +444,10 @@ def _fallback_entries(configured: object) -> tuple[object, ...]:
return ()
def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool:
def _has_fallback_group(ctx: "RoutingContext", requested_model_group: str | None) -> bool:
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
configured: Final[object] = getattr(litellm_router_instance, "fallbacks", None)
configured: Final[object] = ctx.settings().fallbacks
if requested_model_group is None or not configured:
return False
chains: Final = tuple(
@ -462,7 +458,7 @@ def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_
def _should_cooldown_deployment(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str,
exception_status: str | int,
original_exception: Exception,
@ -486,31 +482,29 @@ def _should_cooldown_deployment(
- v1 logic (Legacy): if allowed fails or allowed fail policy set, coolsdown if num fails in this minute > allowed fails
"""
model_group: Final = litellm_router_instance.get_model_group(id=deployment)
model_group: Final = ctx.registry().resolve(deployment, kind="group_of_id")
is_single_deployment_model_group = False
if model_group is not None and len(model_group) == 1:
is_single_deployment_model_group = not (
litellm_router_instance.routing_group_has_alternatives(requested_model_group)
or litellm_router_instance.team_model_has_alternatives(deployment)
ctx.registry().resolve(requested_model_group, kind="routing_group_alternatives")
or ctx.registry().resolve(deployment, kind="team_model_alternatives")
)
is_lone_deployment: Final = is_single_deployment_model_group and not _has_fallback_group(
litellm_router_instance, requested_model_group
)
is_lone_deployment: Final = is_single_deployment_model_group and not _has_fallback_group(ctx, requested_model_group)
exception_status_int: Final = cast_exception_status_to_int(exception_status)
dep_policy, dep_allowed_fails = _get_deployment_cooldown_policy(litellm_router_instance, deployment)
dep_policy, dep_allowed_fails = _get_deployment_cooldown_policy(ctx, deployment)
if (
is_single_deployment_model_group
and exception_status_int == 402
and _resolve_allowed_fails_from_policy(dep_policy, original_exception) is None
and litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None
and ctx.settings().allowed_fails_for(original_exception) is None
):
return False
## CHECK DEPLOYMENT-LEVEL POLICY FIRST (overrides router-level)
if dep_policy is not None or dep_allowed_fails is not None:
return _should_cooldown_based_on_deployment_policy(
litellm_router_instance,
ctx,
deployment,
original_exception,
dep_policy,
@ -519,16 +513,11 @@ def _should_cooldown_deployment(
)
## BASE CASE - single deployment
if (
litellm_router_instance.allowed_fails_policy is None
and _is_allowed_fails_set_on_router(litellm_router_instance=litellm_router_instance) is False
):
if ctx.settings().allowed_fails_policy is None and _is_allowed_fails_set_on_router(ctx=ctx) is False:
num_successes_this_minute: Final = get_deployment_successes_for_current_minute(
litellm_router_instance=litellm_router_instance, deployment_id=deployment
)
num_fails_this_minute: Final = get_deployment_failures_for_current_minute(
litellm_router_instance=litellm_router_instance, deployment_id=deployment
ctx=ctx, deployment_id=deployment
)
num_fails_this_minute: Final = get_deployment_failures_for_current_minute(ctx=ctx, deployment_id=deployment)
total_requests_this_minute: Final = num_successes_this_minute + num_fails_this_minute
percent_fails = 0.0
@ -560,10 +549,10 @@ def _should_cooldown_deployment(
return False
else:
if is_lone_deployment and litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None:
if is_lone_deployment and ctx.settings().allowed_fails_for(original_exception) is None:
return False
return should_cooldown_based_on_allowed_fails_policy(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment=deployment,
original_exception=original_exception,
)
@ -571,8 +560,9 @@ def _should_cooldown_deployment(
return False
@renamed_router_param("litellm_router_instance")
def set_cooldown_deployments(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
original_exception: Exception,
exception_status: str | int,
deployment: str | None = None,
@ -595,7 +585,7 @@ def set_cooldown_deployments(
if (
_should_run_cooldown_logic(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
@ -612,13 +602,13 @@ def set_cooldown_deployments(
verbose_router_logger.debug("Attempting to add %s to cooldown list", deployment)
if _should_cooldown_deployment(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
requested_model_group=requested_model_group,
):
litellm_router_instance.cooldown_cache.add_deployment_to_cooldown(
ctx.health().cooldowns.add_deployment_to_cooldown(
model_id=deployment,
original_exception=original_exception,
exception_status=exception_status_int,
@ -628,7 +618,7 @@ def set_cooldown_deployments(
# Trigger cooldown callback handler
asyncio.create_task(
router_cooldown_event_callback(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
deployment_id=deployment,
exception_status=exception_status,
cooldown_time=time_to_cooldown,
@ -641,15 +631,16 @@ def set_cooldown_deployments(
_set_cooldown_deployments = set_cooldown_deployments
@renamed_router_param("litellm_router_instance")
async def async_get_cooldown_deployments(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
parent_otel_span: Span | None,
) -> list[str]:
"""
Async implementation of 'get_cooldown_deployments'
"""
model_ids: Final = litellm_router_instance.get_model_ids()
cooldown_models: Final = await litellm_router_instance.cooldown_cache.async_get_active_cooldowns(
model_ids: Final = ctx.registry().names(kind="ids")
cooldown_models: Final = await ctx.health().cooldowns.async_get_active_cooldowns(
model_ids=model_ids,
parent_otel_span=parent_otel_span,
)
@ -670,15 +661,16 @@ async def async_get_cooldown_deployments(
_async_get_cooldown_deployments = async_get_cooldown_deployments
@renamed_router_param("litellm_router_instance")
async def async_get_cooldown_deployments_with_debug_info(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
parent_otel_span: Span | None,
) -> list[tuple[str, CooldownCacheValue]]:
"""
Async implementation of 'get_cooldown_deployments'
"""
model_ids: Final = litellm_router_instance.get_model_ids()
cooldown_models: Final = await litellm_router_instance.cooldown_cache.async_get_active_cooldowns(
model_ids: Final = ctx.registry().names(kind="ids")
cooldown_models: Final = await ctx.health().cooldowns.async_get_active_cooldowns(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
@ -689,7 +681,8 @@ async def async_get_cooldown_deployments_with_debug_info(
_async_get_cooldown_deployments_with_debug_info = async_get_cooldown_deployments_with_debug_info
def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel_span: Span | None) -> list[str]:
@renamed_router_param("litellm_router_instance")
def get_cooldown_deployments(ctx: "RoutingContext", parent_otel_span: Span | None) -> list[str]:
"""
Get the list of models being cooled down for this minute
"""
@ -698,9 +691,9 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel
# ----------------------
# Return cooldown models
# ----------------------
model_ids: Final = litellm_router_instance.get_model_ids()
model_ids: Final = ctx.registry().names(kind="ids")
cooldown_models: Final = litellm_router_instance.cooldown_cache.get_active_cooldowns(
cooldown_models: Final = ctx.health().cooldowns.get_active_cooldowns(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
@ -719,28 +712,30 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel
_get_cooldown_deployments = get_cooldown_deployments
def _deployment_api_base(litellm_router_instance: LitellmRouter, model_id: str) -> str | None:
deployment: Final = litellm_router_instance.get_deployment(model_id=model_id)
def _deployment_api_base(ctx: "RoutingContext", model_id: str) -> str | None:
deployment: Final = ctx.registry().deployment(model_id)
api_base: Final = deployment.litellm_params.api_base if deployment else None
return None if api_base is None else mask_api_base_credentials(api_base)
@renamed_router_param("litellm_router_instance")
def get_cooldown_deployment_info(
litellm_router_instance: LitellmRouter, route_model_ids: Collection[str], cooldown_list: Sequence[str]
ctx: "RoutingContext", route_model_ids: Collection[str], cooldown_list: Sequence[str]
) -> tuple[CooldownDeploymentInfo, ...]:
"""
The deployments the router resolved for this request (`route_model_ids`) that are in `cooldown_list`, each
with its masked api_base
"""
return tuple(
CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(litellm_router_instance, model_id))
CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(ctx, model_id))
for model_id in cooldown_list
if model_id in route_model_ids
)
@renamed_router_param("litellm_router_instance")
def should_cooldown_based_on_allowed_fails_policy(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment: str,
original_exception: Exception,
allowed_fails_override: int | None = None,
@ -764,29 +759,21 @@ def should_cooldown_based_on_allowed_fails_policy(
- True if fails exceed the allowed limit (should cooldown)
- False if fails are within the allowed limit (should not cooldown)
"""
allowed_fails_from_policy: Final = litellm_router_instance.get_allowed_fails_from_policy(
exception=original_exception
)
allowed_fails_from_policy: Final = ctx.settings().allowed_fails_for(original_exception)
allowed_fails: Final = (
allowed_fails_override
if allowed_fails_override is not None
else (
allowed_fails_from_policy
if allowed_fails_from_policy is not None
else litellm_router_instance.allowed_fails
)
else (allowed_fails_from_policy if allowed_fails_from_policy is not None else ctx.settings().allowed_fails)
)
cooldown_time: Final = (
cooldown_time_override
if cooldown_time_override is not None
else (litellm_router_instance.cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS)
else (ctx.settings().cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS)
)
base_key: Final = f"deployment:{deployment}:allowed_fails"
cache_key: Final = f"{base_key}:{cache_key_suffix}" if cache_key_suffix else base_key
updated_fails: Final = _increment_allowed_fails(
cache=litellm_router_instance.cache, cache_key=cache_key, ttl=cooldown_time
)
updated_fails: Final = _increment_allowed_fails(cache=ctx.health().store, cache_key=cache_key, ttl=cooldown_time)
return updated_fails > allowed_fails
@ -805,7 +792,7 @@ def _increment_allowed_fails(cache: DualCache, cache_key: str, ttl: float) -> in
def _is_allowed_fails_set_on_router(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
) -> bool:
"""
Check if Router.allowed_fails was passed to the Router or is a Non-default Value
@ -815,11 +802,11 @@ def _is_allowed_fails_set_on_router(
- True if Router.allowed_fails is a Non-default Value
- False if Router.allowed_fails is None, or is the Default Value and was not passed
"""
if litellm_router_instance.allowed_fails is None:
if ctx.settings().allowed_fails is None:
return False
if litellm_router_instance.explicit_allowed_fails is not None:
if ctx.explicit_allowed_fails is not None:
return True
if litellm_router_instance.allowed_fails != litellm.allowed_fails:
if ctx.settings().allowed_fails != litellm.allowed_fails:
return True
return False

View file

@ -13,16 +13,17 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
get_fallback_error_info,
)
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
from litellm.litellm_core_utils.litellm_logging import Logging, same_attempt_marker
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
get_fallback_error_info,
)
from litellm.router_deprecated import renamed_router_param
from litellm.router_utils.cooldown_handlers import (
_first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils
cast_exception_status_to_int,
@ -43,11 +44,7 @@ from litellm.types.router import (
)
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
else:
LitellmRouter = Any
from litellm.router_api import RoutingContext
# Status codes a generic API call's caller-supplied resource id can trigger on its own
# (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health.
@ -57,7 +54,7 @@ _FALLBACK_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
def _trigger_cooldown_for_failed_deployment(
litellm_router: LitellmRouter,
ctx: "RoutingContext",
kwargs: Mapping[str, object],
exception: Exception,
model_call_details: Mapping[str, object] = _NO_MODEL_CALL_DETAILS,
@ -128,7 +125,7 @@ def _trigger_cooldown_for_failed_deployment(
verbose_router_logger.debug("Cannot trigger cooldown for fallback: no failed_deployment_id on exception")
return
deployment: Final = litellm_router.get_deployment(model_id=deployment_id)
deployment: Final = ctx.registry().deployment(deployment_id)
if is_caller_scoped_auth_failure(deployment, exception_status):
verbose_router_logger.debug(
"Not triggering cooldown for fallback deployment %s: caller-scoped OAuth authentication failure.",
@ -138,7 +135,7 @@ def _trigger_cooldown_for_failed_deployment(
# Priority: deployment config > response header > router default, matching
# Router.deployment_callback_on_failure's precedence for the primary path.
deployment_dict: Final = litellm_router.get_model_info(id=deployment_id)
deployment_dict: Final = ctx.registry().deployment_dict(deployment_id)
deployment_cooldown: Final = (
_first_present(
deployment_dict.get("model_info"), deployment_dict.get("litellm_params"), key="cooldown_time"
@ -161,16 +158,16 @@ def _trigger_cooldown_for_failed_deployment(
else (
header_cooldown
if header_cooldown is not None and header_cooldown >= 0
else litellm_router.cooldown_time
else ctx.settings().cooldown_time
)
)
increment_deployment_failures_for_current_minute(
litellm_router_instance=litellm_router,
ctx=ctx,
deployment_id=deployment_id,
)
set_cooldown_deployments(
litellm_router_instance=litellm_router,
ctx=ctx,
exception_status=exception_status,
original_exception=exception,
deployment=deployment_id,
@ -697,14 +694,14 @@ def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object])
async def _is_fallback_target_authorized(
litellm_router: LitellmRouter,
ctx: "RoutingContext",
fallback_entry: str | Mapping[str, object],
original_model_group: str,
kwargs: Mapping[str, object],
*,
attempted: AttemptedFallbackTargets,
) -> bool:
access_check: Final = litellm_router.fallback_access_check
access_check: Final = ctx.settings().fallback_access_check
target: Final = _get_fallback_target_model_group(fallback_entry)
if (
access_check is None
@ -713,7 +710,7 @@ async def _is_fallback_target_authorized(
or target == get_pre_routing_selection(kwargs)
):
return True
if await access_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
if await ctx.fallback_access_allowed(model=target, request_kwargs=kwargs):
return True
verbose_router_logger.info(
"Skipping fallback to model_group = %s: caller is not authorized to call it",
@ -724,18 +721,18 @@ async def _is_fallback_target_authorized(
async def _is_fallback_target_within_budget(
litellm_router: LitellmRouter,
ctx: "RoutingContext",
fallback_entry: str | Mapping[str, object],
original_model_group: str,
kwargs: Mapping[str, object],
*,
attempted: AttemptedFallbackTargets,
) -> bool:
budget_check: Final = litellm_router.fallback_budget_check
budget_check: Final = ctx.settings().fallback_budget_check
target: Final = _get_fallback_target_model_group(fallback_entry)
if budget_check is None or target is None or target == original_model_group:
return True
if await budget_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
if await ctx.fallback_budget_allowed(model=target, request_kwargs=kwargs):
return True
verbose_router_logger.info(
"Skipping fallback to model_group = %s: caller is over budget",
@ -779,9 +776,10 @@ def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
@renamed_router_param("litellm_router", new="litellm_routing_ctx")
async def run_async_fallback(
*args: object,
litellm_router: LitellmRouter,
litellm_routing_ctx: "RoutingContext",
fallback_model_group: list[str],
original_model_group: str,
original_exception: Exception,
@ -798,7 +796,7 @@ async def run_async_fallback(
If all fallback model groups fail, it raises the most recent exception.
Args:
litellm_router: The litellm router instance.
litellm_routing_ctx: The routing context of the router running the fallbacks.
*args: Positional arguments.
fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
original_model_group: The original model group. example: "gpt-3.5-turbo"
@ -817,6 +815,7 @@ async def run_async_fallback(
Raises:
The most recent exception if all fallback model groups fail.
"""
ctx: Final = litellm_routing_ctx
### BASE CASE ### MAX FALLBACK DEPTH REACHED
if fallback_depth >= max_fallbacks:
@ -853,13 +852,9 @@ async def run_async_fallback(
original_model_group,
)
continue
if not await _is_fallback_target_authorized(
litellm_router, mg, original_model_group, kwargs, attempted=attempted
):
if not await _is_fallback_target_authorized(ctx, mg, original_model_group, kwargs, attempted=attempted):
continue
if not await _is_fallback_target_within_budget(
litellm_router, mg, original_model_group, kwargs, attempted=attempted
):
if not await _is_fallback_target_within_budget(ctx, mg, original_model_group, kwargs, attempted=attempted):
continue
attempt_key = fallback_attempt_key(mg)
if attempt_key is not None:
@ -872,7 +867,7 @@ async def run_async_fallback(
attempted.record(attempt_key)
try:
# LOGGING
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
kwargs = ctx.log_retry(kwargs=kwargs, e=original_exception)
verbose_router_logger.info("Falling back to model_group = %s", mask_sensitive_structure(mg))
kwargs.pop("_target_order", None)
if isinstance(mg, str):
@ -894,7 +889,7 @@ async def run_async_fallback(
if include_fallback_errors:
kwargs["include_fallback_errors"] = include_fallback_errors
carry_superseded_attempt_usage(_FALLBACK_KWARGS_ADAPTER.validate_python(kwargs))
response = await litellm_router.async_function_with_fallbacks(*args, **kwargs)
response = await ctx.with_fallbacks(*args, **kwargs)
verbose_router_logger.info("Successful fallback b/w models.")
response = add_fallback_headers_to_response(
response=response,
@ -922,7 +917,7 @@ async def run_async_fallback(
)
) is not None:
_trigger_cooldown_for_failed_deployment(
litellm_router=litellm_router,
ctx=ctx,
kwargs=kwargs,
exception=e,
model_call_details=failure_callbacks_blocked_logging_obj.model_call_details,

View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
from litellm._logging import redact_secrets, verbose_router_logger
from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH
from litellm.router_deprecated import renamed_router_param
from litellm.router_utils.cooldown_handlers import (
async_get_cooldown_deployments_with_debug_info,
get_cooldown_deployment_info,
@ -13,17 +14,16 @@ from litellm.types.router import RouterRateLimitError
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.router import Router as _Router
from litellm.router_api import RoutingContext
LitellmRouter = _Router
Span = _Span
else:
LitellmRouter = Any
Span = Any
@renamed_router_param("litellm_router_instance")
async def send_llm_exception_alert(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
request_kwargs: dict,
error_traceback_str: str,
original_exception,
@ -33,19 +33,14 @@ async def send_llm_exception_alert(
Sends a Slack / MS Teams alert for the LLM API call failure. Only if router.slack_alerting_logger is set.
Parameters:
litellm_router_instance (_Router): The LitellmRouter instance.
ctx (RoutingContext): The router's routing context.
original_exception (Any): The original exception that occurred.
Returns:
None
"""
if litellm_router_instance is None:
return
if not hasattr(litellm_router_instance, "slack_alerting_logger"):
return
if litellm_router_instance.slack_alerting_logger is None:
slack_alerting_logger: Final = ctx.observers().slack_alerting_logger
if slack_alerting_logger is None:
return
if "proxy_server_request" in request_kwargs:
@ -62,7 +57,7 @@ async def send_llm_exception_alert(
# Redact secrets before sending to external service (Slack / MS Teams)
exception_str = redact_secrets(exception_str)
await litellm_router_instance.slack_alerting_logger.send_alert(
await slack_alerting_logger.send_alert(
message=f"LLM API call failed: `{exception_str}`",
level="High",
alert_type=AlertType.llm_exceptions,
@ -70,8 +65,9 @@ async def send_llm_exception_alert(
)
@renamed_router_param("litellm_router_instance")
async def async_raise_no_deployment_exception(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
model: str,
parent_otel_span: Span | None,
route_model_ids: Collection[str] | None = None,
@ -80,12 +76,12 @@ async def async_raise_no_deployment_exception(
Raises a RouterRateLimitError if no deployment is found for the given model.
"""
verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model)
model_ids: Final = litellm_router_instance.get_model_ids(model_name=model)
_cooldown_time: Final = litellm_router_instance.cooldown_cache.get_min_cooldown(
model_ids: Final = ctx.registry().names(model, kind="ids")
_cooldown_time: Final = ctx.health().cooldowns.get_min_cooldown(
model_ids=model_ids, parent_otel_span=parent_otel_span
)
_cooldown_list: Final = await async_get_cooldown_deployments_with_debug_info(
litellm_router_instance=litellm_router_instance,
ctx=ctx,
parent_otel_span=parent_otel_span,
)
verbose_router_logger.info(
@ -96,12 +92,10 @@ async def async_raise_no_deployment_exception(
return RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
enable_pre_call_checks=ctx.settings().enable_pre_call_checks,
cooldown_list=cooldown_list_ids,
model_ids=model_ids,
cooldown_deployments=(
None
if route_model_ids is None
else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids)
None if route_model_ids is None else get_cooldown_deployment_info(ctx, route_model_ids, cooldown_list_ids)
),
)

View file

@ -14,8 +14,7 @@ from litellm import verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
HEALTH_CHECKS_TARGET: Final = "health_checks"
from litellm.constants import HEALTH_CHECKS_TARGET
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span

View file

@ -13,8 +13,8 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG
from litellm.litellm_core_utils.rules import Rules
from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
from litellm.utils import supports_assistant_prefill

View file

@ -30,8 +30,8 @@ from litellm.caching.affinity_cache import (
from litellm.caching.dual_cache import DualCache
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import CallTypes

View file

@ -50,14 +50,15 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_content_of_block,
strip_encrypted_reasoning_from_messages,
)
from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import Deployment
from litellm.utils import get_order_filtered_deployments
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -74,14 +75,17 @@ class EncryptedContentAffinityCheck(CustomLogger):
per-model group ``model_group_affinity_config``.
"""
router: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
@renamed_router_param("router")
def __init__(
self,
router: Optional["Router"] = None,
ctx: Optional["RoutingContext"] = None,
enable_global_affinity: bool = True,
model_group_affinity_config: dict[str, list[str]] | None = None,
) -> None:
super().__init__()
self.router = router
self.ctx = ctx
self.enable_global_affinity = enable_global_affinity
self.model_group_affinity_config: dict[str, list[str]] = model_group_affinity_config or {}
@ -254,9 +258,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
deployment's ``(api_base, api_key)``, alongside the originating
deployment object (or ``None`` if it was removed / router unavailable).
"""
if self.router is None:
if self.ctx is None:
return [], None
originating: Final = self.router.get_deployment(model_id=model_id)
originating: Final = self.ctx.registry().deployment(model_id)
if originating is None:
return [], None
boundary: Final = self._encryption_boundary_key(originating.litellm_params.model_dump(exclude_none=True))
@ -268,8 +272,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
return matches, originating
@staticmethod
@renamed_router_param("router")
def strip_reasoning_the_targets_cannot_decrypt(
router: "Router | None",
ctx: "RoutingContext | None",
request_input: object,
anthropic_messages: object,
target_deployments: Sequence[Mapping[str, object]],
@ -302,9 +307,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
return False
if origin_model_id in target_ids:
return True
if router is None:
if ctx is None:
return False
origin: Final = router.get_deployment(model_id=origin_model_id)
origin: Final = ctx.registry().deployment(origin_model_id)
origin_boundary: Final = (
EncryptedContentAffinityCheck._encryption_boundary_key(
origin.litellm_params.model_dump(exclude_none=True)
@ -419,7 +424,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
)
request_kwargs["_encrypted_content_affinity_pinned"] = True
self.strip_reasoning_the_targets_cannot_decrypt(
self.router, request_input, anthropic_messages, (deployment,), unmarked_origin=model_id
self.ctx, request_input, anthropic_messages, (deployment,), unmarked_origin=model_id
)
return [deployment]
@ -437,7 +442,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
)
request_kwargs["_encrypted_content_affinity_pinned"] = True
self.strip_reasoning_the_targets_cannot_decrypt(
self.router, request_input, anthropic_messages, boundary_matches, unmarked_origin=model_id
self.ctx, request_input, anthropic_messages, boundary_matches, unmarked_origin=model_id
)
return boundary_matches

View file

@ -17,9 +17,9 @@ from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.utils import (

View file

@ -10,16 +10,13 @@ get_deployment_successes_for_current_minute
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
from litellm.router_deprecated import renamed_router_param
if TYPE_CHECKING:
from litellm.router import Router as _Router
LitellmRouter = _Router
else:
LitellmRouter = Any
from litellm.router_api import RoutingContext
_METADATA_CHANNELS: Final = ("litellm_metadata", "metadata")
@ -42,15 +39,16 @@ def get_counted_usage_tokens(litellm_params: Mapping[str, object]) -> int | None
return counted if isinstance(counted, int) and not isinstance(counted, bool) else None
@renamed_router_param("litellm_router_instance")
def increment_deployment_successes_for_current_minute(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment_id: str,
) -> str:
"""
In-Memory: Increments the number of successes for the current minute for a deployment_id
"""
key: Final = f"{deployment_id}:successes"
litellm_router_instance.cache.increment_cache(
ctx.health().store.increment_cache(
local_only=True,
key=key,
value=1,
@ -59,15 +57,16 @@ def increment_deployment_successes_for_current_minute(
return key
@renamed_router_param("litellm_router_instance")
def increment_deployment_failures_for_current_minute(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment_id: str,
):
"""
In-Memory: Increments the number of failures for the current minute for a deployment_id
"""
key: Final = f"{deployment_id}:fails"
litellm_router_instance.cache.increment_cache(
ctx.health().store.increment_cache(
local_only=True,
key=key,
value=1,
@ -75,8 +74,9 @@ def increment_deployment_failures_for_current_minute(
)
@renamed_router_param("litellm_router_instance")
def get_deployment_successes_for_current_minute(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment_id: str,
) -> int:
"""
@ -86,7 +86,7 @@ def get_deployment_successes_for_current_minute(
"""
key: Final = f"{deployment_id}:successes"
return (
litellm_router_instance.cache.get_cache(
ctx.health().store.get_cache(
local_only=True,
key=key,
)
@ -94,8 +94,9 @@ def get_deployment_successes_for_current_minute(
)
@renamed_router_param("litellm_router_instance")
def get_deployment_failures_for_current_minute(
litellm_router_instance: LitellmRouter,
ctx: "RoutingContext",
deployment_id: str,
) -> int:
"""
@ -105,7 +106,7 @@ def get_deployment_failures_for_current_minute(
"""
key: Final = f"{deployment_id}:fails"
return (
litellm_router_instance.cache.get_cache(
ctx.health().store.get_cache(
local_only=True,
key=key,
)

View file

@ -21,13 +21,14 @@ from litellm._internal_context import service_target
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_batch import BatchResult, active_request_redis_batches
from litellm.router_deprecated import renamed_router_param
from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2, PrefetchedUsage
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCache
if TYPE_CHECKING:
from opentelemetry.trace import Span
from litellm.router import Router
from litellm.router_api import RoutingContext
ROUTER_COOLDOWNS_USAGE_TARGET: Final = "router_cooldowns_usage"
@ -93,23 +94,24 @@ class RoutingPrefetch:
raise
@staticmethod
@renamed_router_param("litellm_router_instance")
def arm(
litellm_router_instance: "Router",
ctx: "RoutingContext",
usage_selector: LowestTPMLoggingHandler_v2 | None,
deployments: list,
) -> None:
request: Final = active_request_redis_batches()
redis_cache: Final = litellm_router_instance.cache.redis_cache
redis_cache: Final = ctx.health().store.redis_cache
if request is None or redis_cache is None or _PREFETCH_SLOT in request.prefetched:
return
cooldown_keys: Final = tuple(
CooldownCache.get_cooldown_cache_key(model_id) for model_id in litellm_router_instance.get_model_ids()
CooldownCache.get_cooldown_cache_key(model_id) for model_id in ctx.registry().names(kind="ids")
)
usage_keys: Final = (
() if usage_selector is None else tuple(itertools.chain(*usage_selector.usage_counter_keys(deployments)))
)
keys: Final = (*cooldown_keys, *usage_keys)
cooldown_store: Final = litellm_router_instance.cooldown_cache.cooldown_store
cooldown_store: Final = ctx.health().cooldowns.cooldown_store
cooldown_due, cooldown_previous = cooldown_store.reserve_redis_batch_reads(cooldown_keys)
usage_cache: Final = None if usage_selector is None else usage_selector.router_cache
usage_reservation: Final = None if usage_cache is None else usage_cache.reserve_redis_batch_reads(usage_keys)
@ -184,9 +186,10 @@ class RoutingReadBatch:
return RoutingReadBatch(usage_selector=selector)
return RoutingReadBatch(usage_selector=None) if RoutingPrefetch.armed() else None
@renamed_router_param("litellm_router_instance")
async def async_get_cooldown_deployments(
self,
litellm_router_instance: "Router",
ctx: "RoutingContext",
healthy_deployments: list,
parent_otel_span: "Span | None",
) -> list[str]:
@ -194,14 +197,14 @@ class RoutingReadBatch:
`_async_get_cooldown_deployments`, with the strategy's tpm/rpm counters for
`healthy_deployments` fetched in the same MGET and kept as `prefetched_usage`.
"""
model_ids: Final = litellm_router_instance.get_model_ids()
model_ids: Final = ctx.registry().names(kind="ids")
cooldown_keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
selector: Final = self.usage_selector
usage_keys: Final = (
() if selector is None else tuple(itertools.chain(*selector.usage_counter_keys(healthy_deployments)))
)
reads: Final = (
(litellm_router_instance.cooldown_cache.cooldown_store, cooldown_keys),
(ctx.health().cooldowns.cooldown_store, cooldown_keys),
*(() if selector is None else ((selector.router_cache, list(usage_keys)),)),
)
with service_target(_routing_read_target(cooldown_keys, usage_keys)):
@ -216,9 +219,7 @@ class RoutingReadBatch:
values=None if usage_values is None else MappingProxyType(dict(zip(usage_keys, usage_values))),
)
cooldown_models: Final = litellm_router_instance.cooldown_cache.active_cooldowns_from_results(
model_ids, cooldown_results
)
cooldown_models: Final = ctx.health().cooldowns.active_cooldowns_from_results(model_ids, cooldown_results)
verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models)
return [model_id for model_id, _ in cooldown_models]

View file

@ -10,16 +10,15 @@ import traceback
from collections.abc import Callable
from functools import partial
from types import MappingProxyType
from typing import Any, Final, Protocol
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_router_logger
from litellm.router_api import ChangeSet
from litellm.router_deprecated import renamed_router_param
from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict
class _SearchToolsRouter(Protocol):
"""The one router attribute the search-tool helpers read and replace."""
search_tools: list[SearchToolTypedDict]
if TYPE_CHECKING:
from litellm.router_api import RoutingContext
class SearchAPIRouter:
@ -52,14 +51,15 @@ class SearchAPIRouter:
return resolved_api_key, resolved_api_base
@staticmethod
async def update_router_search_tools(router_instance: _SearchToolsRouter, search_tools: list):
@renamed_router_param("router_instance")
async def update_router_search_tools(ctx: "RoutingContext", search_tools: list):
"""
Update the router with search tools from the database.
This method is called by a cron job to sync search tools from DB to router.
Args:
router_instance: The Router instance to update
ctx: The routing context of the router to update
search_tools: List of search tool configurations from the database
"""
try:
@ -78,7 +78,7 @@ class SearchAPIRouter:
router_search_tools.append(router_search_tool)
# Update the router's search_tools list
router_instance.search_tools = router_search_tools
ctx.apply(ChangeSet(assign={"search_tools": router_search_tools}))
verbose_router_logger.info("Successfully updated router with %s search tool(s)", len(router_search_tools))
@ -87,15 +87,16 @@ class SearchAPIRouter:
raise e
@staticmethod
@renamed_router_param("router_instance")
def get_matching_search_tools(
router_instance: _SearchToolsRouter,
ctx: "RoutingContext",
search_tool_name: str,
) -> list:
"""
Get all search tools matching the given name.
Args:
router_instance: The Router instance
ctx: The routing context of the router
search_tool_name: Name of the search tool to find
Returns:
@ -105,7 +106,7 @@ class SearchAPIRouter:
ValueError: If no matching search tools are found
"""
matching_tools: Final = [
tool for tool in router_instance.search_tools if tool.get("search_tool_name") == search_tool_name
tool for tool in ctx.settings().search_tools if tool.get("search_tool_name") == search_tool_name
]
if not matching_tools:
@ -114,8 +115,9 @@ class SearchAPIRouter:
return matching_tools
@staticmethod
@renamed_router_param("router_instance", new="litellm_routing_ctx")
async def async_search_with_fallbacks(
router_instance: Any,
litellm_routing_ctx: "RoutingContext",
original_function: Callable,
**kwargs,
):
@ -124,13 +126,14 @@ class SearchAPIRouter:
Reuses the router's retry/fallback infrastructure.
Args:
router_instance: The Router instance
litellm_routing_ctx: The routing context of the router
original_function: The original litellm.asearch function
**kwargs: Search parameters including search_tool_name, query, etc.
Returns:
SearchResponse from the search API
"""
ctx: Final = litellm_routing_ctx
try:
search_tool_name: Final = kwargs.get("search_tool_name", kwargs.get("model"))
@ -140,20 +143,20 @@ class SearchAPIRouter:
# Set up kwargs for the fallback system
kwargs["model"] = search_tool_name # Use model field for compatibility with fallback system
kwargs["original_generic_function"] = original_function
# Bind router_instance to the helper method using partial
# Bind ctx to the helper method using partial
kwargs["original_function"] = partial(
SearchAPIRouter.async_search_with_fallbacks_helper,
router_instance=router_instance,
litellm_routing_ctx=ctx,
)
# Update kwargs before fallbacks (for logging, metadata, etc)
router_instance._update_kwargs_before_fallbacks(
ctx.update_kwargs_before_fallbacks(
model=search_tool_name,
kwargs=kwargs,
metadata_variable_name="litellm_metadata",
)
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools]
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in ctx.settings().search_tools]
verbose_router_logger.debug(
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s",
search_tool_name,
@ -161,7 +164,7 @@ class SearchAPIRouter:
)
# Use the existing retry/fallback infrastructure
response: Final = await router_instance.async_function_with_fallbacks(**kwargs)
response: Final = await ctx.with_fallbacks(**kwargs)
return response
except Exception as e:
@ -169,7 +172,7 @@ class SearchAPIRouter:
asyncio.create_task(
send_llm_exception_alert(
litellm_router_instance=router_instance,
ctx=ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
@ -178,8 +181,9 @@ class SearchAPIRouter:
raise e
@staticmethod
@renamed_router_param("router_instance", new="litellm_routing_ctx")
async def async_search_with_fallbacks_helper(
router_instance: _SearchToolsRouter,
litellm_routing_ctx: "RoutingContext",
model: str,
original_generic_function: Callable,
**kwargs,
@ -189,7 +193,7 @@ class SearchAPIRouter:
Called by async_function_with_fallbacks for each retry attempt.
Args:
router_instance: The Router instance
litellm_routing_ctx: The routing context of the router
model: The search tool name (passed as model for compatibility)
original_generic_function: The original litellm.asearch function
**kwargs: Search parameters
@ -197,12 +201,13 @@ class SearchAPIRouter:
Returns:
SearchResponse from the selected search provider
"""
ctx: Final = litellm_routing_ctx
search_tool_name: Final = model # model field contains the search_tool_name
try:
# Find matching search tools
matching_tools: Final = SearchAPIRouter.get_matching_search_tools(
router_instance=router_instance,
ctx=ctx,
search_tool_name=search_tool_name,
)

View file

@ -103,7 +103,7 @@ def finalize(
update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time)
cache_key: Final = logger.model_call_details.get("cache_key")
if logger.model_call_details.get("cache_hit") is True and isinstance(cache_key, str):
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
hidden: Final = get_hidden_params_dict(response, create=True)
hidden.update({"cache_key": cache_key, "cache_hit": True})

View file

@ -1,6 +1,6 @@
from typing import Final, TypeVar
from litellm.router_utils.add_retry_fallback_headers import (
from litellm.litellm_core_utils.add_retry_fallback_headers import (
_add_headers_to_response, # pyright: ignore[reportPrivateUsage] # reuse the proxy's identity-preserving response metadata writer
get_hidden_params_dict,
)

View file

@ -17,8 +17,8 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.router_api import AttemptedFallbackTargets
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
from litellm.types.caching import DynamicCacheControl
from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage
from litellm.types.proxy.guardrails.guardrail_hooks.headroom import HeadroomRequestContext

View file

@ -37,7 +37,7 @@ from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router_weights import RouterWeights
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext as RouterContext
from .completion import CompletionRequest
from .embedding import EmbeddingRequest
@ -1276,7 +1276,7 @@ class FallbackAccessCheck(Protocol):
rejects, so a fallback can never reach a model the caller could not have requested directly.
"""
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ...
class FallbackBudgetCheck(Protocol):
@ -1289,7 +1289,7 @@ class FallbackBudgetCheck(Protocol):
and skips targets it rejects, leaving the free attempt itself untouched.
"""
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ...
class AutoRouterCapabilityLimit(Protocol):

View file

@ -363,6 +363,10 @@ if TYPE_CHECKING:
get_llm_provider,
is_non_openai_azure_model,
)
from litellm.litellm_core_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy,
reset_retry_policy,
)
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
@ -455,10 +459,6 @@ if TYPE_CHECKING:
)
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.router_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy,
reset_retry_policy,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_API_ONLY_HEADERS,
AnthropicThinkingParam,
@ -5258,7 +5258,7 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c
def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tuple[str, ...]:
from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
return deployment_reasoning_effort_levels(model_info) or ()

View file

@ -103,7 +103,7 @@ def percentile(values: Sequence[float], quantile: float) -> float:
def verify(response: object, expected_transport: str) -> None:
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
pages: Final = getattr(response, "pages", ())
if len(pages) != 1 or getattr(pages[0], "markdown", None) != EXPECTED_MARKDOWN:

View file

@ -198,11 +198,11 @@ def main():
router_file = [
"./litellm/router.py",
"./litellm/litellm_core_utils/batch_utils.py",
"./litellm/router_utils/pattern_match_deployments.py",
"./litellm/litellm_core_utils/pattern_match_deployments.py",
]
# router_file = [
# "../../litellm/router.py",
# "../../litellm/router_utils/pattern_match_deployments.py",
# "../../litellm/litellm_core_utils/pattern_match_deployments.py",
# "../../litellm/litellm_core_utils/batch_utils.py",
# ] ## LOCAL TESTING
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere

View file

@ -1,7 +1,6 @@
{
"_doc": "Router members and litellm.router_utils modules used outside the router. Shrink-only: see tests/code_coverage_tests/router_surface_check.py.",
"members_outside": [
"_get_client",
"a_add_message",
"aadapter_completion",
"aanthropic_messages",
@ -38,79 +37,14 @@
"avector_store_search",
"completion",
"embedding",
"get_routing_group",
"get_valid_args",
"health",
"observe",
"priority_header_update",
"registry",
"retire",
"schedule_acompletion",
"select",
"settings",
"team_public_model_names_of"
"settings"
],
"members_router_family": [
"_count_pre_call_check_tokens",
"_pre_call_checks",
"_update_kwargs_before_fallbacks",
"acompletion",
"adecisions",
"aembedding",
"allowed_fails",
"allowed_fails_policy",
"aresponses",
"async_function_with_fallbacks",
"async_get_healthy_deployments",
"cache",
"cached_deployment_model_info",
"cooldown_cache",
"cooldown_time",
"default_max_parallel_requests",
"deployments_named",
"disable_cooldowns",
"embedding",
"enable_pre_call_checks",
"enable_tag_filtering",
"explicit_allowed_fails",
"fallback_access_check",
"fallback_budget_check",
"fallbacks",
"get_all_deployments",
"get_allowed_fails_from_policy",
"get_deployment",
"get_model_group",
"get_model_ids",
"get_model_info",
"get_model_list",
"get_router_model_info",
"log_retry",
"model_list",
"provider_default_deployment_ids",
"routing_group_has_alternatives",
"routing_strategy",
"search_tools",
"set_custom_routing_strategy",
"slack_alerting_logger",
"tag_filtering_match_any",
"tag_routing_prefix",
"team_model_has_alternatives"
],
"router_utils_imports_outside": [
"litellm.router_utils.access_windows",
"litellm.router_utils.add_retry_fallback_headers",
"litellm.router_utils.clientside_credential_handler",
"litellm.router_utils.common_utils",
"litellm.router_utils.cooldown_handlers",
"litellm.router_utils.deployment_pricing",
"litellm.router_utils.fallback_event_handlers",
"litellm.router_utils.get_retry_from_policy",
"litellm.router_utils.health_state_cache",
"litellm.router_utils.pattern_match_deployments",
"litellm.router_utils.ptu_shares",
"litellm.router_utils.reasoning_effort_capability",
"litellm.router_utils.router_callbacks.track_deployment_metrics",
"litellm.router_utils.routing_groups",
"litellm.router_utils.search_api_router"
]
"members_router_family": [],
"router_utils_imports_outside": []
}

View file

@ -143,6 +143,6 @@ async def test_string_valued_fallbacks_still_cool_down_the_failing_primary(fallb
model="primary",
messages=[{"role": "user", "content": f"cooldown {uuid.uuid4().hex}"}],
)
cooled: Final = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None)
cooled: Final = await async_get_cooldown_deployments(ctx=router._core_ctx, parent_otel_span=None)
assert tuple(cooled) == ("primary-deployment",)

View file

@ -7,7 +7,7 @@ from typing import Final, Literal
from pydantic import BaseModel, TypeAdapter
import litellm
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule
from litellm.rust_bridge.chat_completions.entrypoints import NATIVE_ACOMPLETION

View file

@ -10,6 +10,7 @@ from litellm.caching.caching import Cache, CacheMode
from litellm.caching.caching_handler import (
_PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request
)
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
from litellm.proxy.hooks.model_max_budget_limiter import (
PROXY_VirtualKeyModelMaxBudgetLimiter,
@ -17,7 +18,6 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
from litellm.proxy.utils import InternalUsageCache
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule
from litellm.rust_bridge.configuration import Rollout

View file

@ -11,7 +11,7 @@ from litellm._v2.cache import NativeBackend
from litellm.caching.caching_handler import (
_PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request
)
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.types.utils import ModelResponse
from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE
from tests.test_litellm_rust.responses.data import RESPONSES_MODEL, RESPONSES_RESPONSE, RESPONSES_TEXT

View file

@ -1,6 +1,6 @@
from typing import Final
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from tests.test_litellm_rust.support.recording_server import RecordingServer

View file

@ -66,7 +66,7 @@ _SDK_CONTRACT = textwrap.dedent(
import litellm
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
class Handler(BaseHTTPRequestHandler):
def do_POST(self):

View file

@ -29,6 +29,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROXY_MaxParallelRequestsHandler_v3,
)
from litellm.proxy.utils import InternalUsageCache
from litellm.router_api import RoutingContext
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCache
from litellm.router_utils.routing_read_batch import (
ROUTER_COOLDOWNS_USAGE_TARGET,
@ -390,7 +391,7 @@ async def test_a_prefetch_with_incomplete_usage_keys_releases_cooldown_reservati
)
with request_redis_batch_scope():
RoutingPrefetch.arm(router, router.lowesttpm_logger_v2, router.model_list[:1])
RoutingPrefetch.arm(RoutingContext(router), router.lowesttpm_logger_v2, router.model_list[:1])
deployment: Final = await router.async_get_available_deployment(
model=_MODEL_GROUP, messages=[{"role": "user", "content": "ping"}], request_kwargs={}
)

View file

@ -241,21 +241,21 @@ async def test_router_cooldown_event_callback():
Ensures that the router_cooldown_event_callback function correctly logs the cooldown event to the PrometheusLogger
"""
# Mock Router instance
mock_router = MagicMock()
# Mock routing context
mock_ctx = MagicMock()
mock_deployment = {
"litellm_params": {"model": "gpt-5-mini"},
"model_name": "gpt-5-mini",
"model_info": ModelInfo(id="test-model-id"),
}
mock_router.get_deployment.return_value = mock_deployment
mock_ctx.registry.return_value.deployment.return_value = mock_deployment
# Create a real PrometheusLogger instance
prometheus_logger = CustomPrometheusLogger()
litellm.callbacks = [prometheus_logger]
await router_cooldown_event_callback(
litellm_router_instance=mock_router,
ctx=mock_ctx,
deployment_id="test-deployment",
exception_status="429",
cooldown_time=60.0,
@ -263,8 +263,8 @@ async def test_router_cooldown_event_callback():
await asyncio.sleep(0.5)
# Assert that the router's get_deployment method was called
mock_router.get_deployment.assert_called_once_with(model_id="test-deployment")
# Assert that the router's deployment lookup was called
mock_ctx.registry.return_value.deployment.assert_called_once_with("test-deployment")
print(
"prometheus_logger.deployment_complete_outages",
@ -301,21 +301,21 @@ async def test_router_cooldown_event_callback_no_prometheus():
Ensures that the router_cooldown_event_callback function does not raise an error when no PrometheusLogger is found
"""
# Mock Router instance
mock_router = MagicMock()
# Mock routing context
mock_ctx = MagicMock()
mock_deployment = {
"litellm_params": {"model": "gpt-5-mini"},
"model_name": "gpt-5-mini",
"model_info": ModelInfo(id="test-model-id"),
}
mock_router.get_deployment.return_value = mock_deployment
mock_ctx.registry.return_value.deployment.return_value = mock_deployment
await router_cooldown_event_callback(
litellm_router_instance=mock_router,
ctx=mock_ctx,
deployment_id="test-deployment",
exception_status="429",
cooldown_time=60.0,
)
# Assert that the router's get_deployment method was called
mock_router.get_deployment.assert_called_once_with(model_id="test-deployment")
# Assert that the router's deployment lookup was called
mock_ctx.registry.return_value.deployment.assert_called_once_with("test-deployment")

View file

@ -6879,7 +6879,7 @@ def test_rebuilding_a_priced_stream_bills_the_margin_once(monkeypatch: pytest.Mo
def test_chunk_creator_keeps_provider_model_private_across_stream():
from litellm.router_utils.add_retry_fallback_headers import (
from litellm.litellm_core_utils.add_retry_fallback_headers import (
get_hidden_params_dict,
)
@ -6940,7 +6940,7 @@ def test_chunk_creator_keeps_provider_model_private_across_stream():
def test_assembled_stream_uses_later_provider_model_for_cost(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.router_utils.add_retry_fallback_headers import (
from litellm.litellm_core_utils.add_retry_fallback_headers import (
get_hidden_params_dict,
)

Some files were not shown because too many files have changed in this diff Show more