mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(router): route the router family through RoutingContext and close the outside surface
Strategies, pre-call checks, cooldown and fallback helpers, and the auto, complexity and quality sub-routers now take a RoutingContext instead of the Router, and read the router only through registry(), settings(), health() and the context's forwarding methods. Router-free helpers move from router_utils to litellm_core_utils, outside callers of router state switch to entry points, and the surface allowlist shrinks to the endpoint methods plus the entry points
This commit is contained in:
parent
5029d027a1
commit
3d33aed3cd
156 changed files with 2514 additions and 2082 deletions
|
|
@ -1469,11 +1469,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
|
|||
"AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"),
|
||||
"HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"),
|
||||
"get_num_retries_from_retry_policy": (
|
||||
"litellm.router_utils.get_retry_from_policy",
|
||||
"litellm.litellm_core_utils.get_retry_from_policy",
|
||||
"get_num_retries_from_retry_policy",
|
||||
),
|
||||
"reset_retry_policy": (
|
||||
"litellm.router_utils.get_retry_from_policy",
|
||||
"litellm.litellm_core_utils.get_retry_from_policy",
|
||||
"reset_retry_policy",
|
||||
),
|
||||
"get_coroutine_checker": (
|
||||
|
|
|
|||
|
|
@ -2046,6 +2046,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
|
|||
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
|
||||
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
|
||||
AUTH_OBJECTS_TARGET: Final = "auth_objects"
|
||||
HEALTH_CHECKS_TARGET: Final = "health_checks"
|
||||
DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)
|
||||
SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
|
||||
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
|
|||
from litellm.repositories.table_repositories import EndUserRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
|
||||
from litellm.router_api import deployment_metadata
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.integrations.prometheus import *
|
||||
from litellm.types.integrations.prometheus import (
|
||||
|
|
@ -3022,8 +3022,8 @@ class PrometheusLogger(CustomLogger):
|
|||
).get("custom_llm_provider")
|
||||
if isinstance(provider, str) and provider:
|
||||
return provider
|
||||
deployment_metadata: Final = find_deployment_metadata(litellm_params)
|
||||
return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
|
||||
picked_deployment: Final = deployment_metadata(litellm_params)
|
||||
return _deployment_model_provider(picked_deployment.get("deployment") if picked_deployment else None)
|
||||
|
||||
@staticmethod
|
||||
def _extract_api_provider_from_exception(exception: Exception) -> str | None:
|
||||
|
|
@ -5209,7 +5209,7 @@ def _tag_matches_wildcard_configured_pattern(tags: Sequence[str], configured_tag
|
|||
"""
|
||||
import re
|
||||
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
pattern_router: Final = PatternMatchRouter()
|
||||
regex_pattern: Final = pattern_router.pattern_to_regex(configured_tag)
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@ from typing import Final
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
filter_internal_params,
|
||||
safe_deep_copy,
|
||||
)
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
)
|
||||
|
||||
from .asyncify import run_async_function
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segmen
|
|||
|
||||
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
|
||||
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
provider_prefix: Final = f"{custom_llm_provider}/"
|
||||
segments: Final = wildcard.removeprefix(provider_prefix).split("/")
|
||||
|
|
|
|||
|
|
@ -12,13 +12,13 @@ from datetime import date
|
|||
from typing import Final, Generic, TypeAlias, TypeVar
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import parsed_ptu_shares, ptu_terms
|
||||
from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, team_may_use_deployment
|
||||
from litellm.llms.azure.ptu_capacity import (
|
||||
PTUCapacity,
|
||||
deployment_ptu_capacity,
|
||||
is_azure_deployment,
|
||||
is_str_keyed_mapping,
|
||||
)
|
||||
from litellm.router_utils.common_utils import get_request_team_id, team_may_use_deployment
|
||||
|
||||
_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
|
||||
WildcardRoute: TypeAlias = Callable[[str], Sequence[Mapping[str, object]]]
|
||||
|
|
@ -56,9 +56,9 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.litellm_core_utils.url_utils import safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicContentParamSource,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,13 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
RequestedReasoningEffort,
|
||||
deployment_reasoning_effort_levels,
|
||||
keeps_effort,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
strip_provider_specific_fields_from_anthropic_messages,
|
||||
|
|
@ -31,13 +38,6 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
RequestedReasoningEffort,
|
||||
deployment_reasoning_effort_levels,
|
||||
keeps_effort,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AllAnthropicToolsValues
|
||||
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import litellm
|
|||
import litellm.constants as _c
|
||||
from litellm.litellm_core_utils.url_utils import validate_url
|
||||
from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
|
||||
from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure
|
||||
from litellm.router_api import mark_advisor_orchestration_failure
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ def normalize_reasoning_effort_value(
|
|||
if chain is None:
|
||||
return effort
|
||||
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
deployment_rounds_reasoning_effort,
|
||||
resolve_supported_reasoning_efforts,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
SUPPORTED_OPENAI_PARAMS: Final = (
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from pydantic import TypeAdapter
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
CompactionProtocol: TypeAlias = Literal["chat", "messages"]
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -32,7 +32,9 @@ def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompac
|
|||
return compaction if compaction.supports_native_compaction(params) else None
|
||||
|
||||
|
||||
async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
async def dispatch(
|
||||
router: RoutingContext, protocol: CompactionProtocol, payload: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
if protocol == "messages":
|
||||
return _MAPPING.validate_python(
|
||||
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
|
||||
|
|
@ -45,4 +47,4 @@ async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mappin
|
|||
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
|
||||
),
|
||||
)
|
||||
return _MAPPING.validate_python(response.model_dump())
|
||||
return _MAPPING.validate_python(response.model_dump()) # pyright: ignore[reportAttributeAccessIssue] # stream=False: acompletion returns a ModelResponse
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
strip_litellm_internal_message_fields,
|
||||
strip_none_values_from_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
declared_reasoning_efforts_for_model,
|
||||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.openai.chat.gpt_transformation import (
|
||||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
declared_reasoning_efforts_for_model,
|
||||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.mistral import (
|
||||
MistralCacheKeyParams,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -54,7 +55,6 @@ from litellm.llms.oci.common_utils import (
|
|||
sign_oci_request,
|
||||
validate_oci_environment,
|
||||
)
|
||||
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.types.llms.oci import (
|
||||
CohereChatRequest,
|
||||
OCIChatRequestPayload,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields
|
||||
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
messages_without_pdf_file_parts,
|
||||
)
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
model_group_from_metadata,
|
||||
reasoning_effort_needs_allowing,
|
||||
resolve_chat_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import (
|
||||
get_configured_request_timeout,
|
||||
)
|
||||
|
|
@ -144,11 +149,6 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
)
|
||||
from litellm.realtime_api.main import realtime_health_check
|
||||
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
model_group_from_metadata,
|
||||
reasoning_effort_needs_allowing,
|
||||
resolve_chat_reasoning,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.completion import (
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ from litellm.repositories.table_repositories import (
|
|||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RegistryReader
|
||||
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
|
||||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
|
|
@ -525,7 +526,7 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
|
||||
def _get_router_zero_cost_cache(llm_router: RegistryReader) -> dict[str, bool] | None:
|
||||
"""
|
||||
Return the router's per-instance zero-cost cache, or ``None`` for objects
|
||||
that don't expose one (e.g. ``MagicMock`` stand-ins in unit tests).
|
||||
|
|
@ -541,7 +542,7 @@ def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
|
|||
return cache if isinstance(cache, dict) else None
|
||||
|
||||
|
||||
def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) -> bool:
|
||||
def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader | None) -> bool:
|
||||
"""
|
||||
Check if a model has zero cost (no configured pricing).
|
||||
|
||||
|
|
@ -667,7 +668,7 @@ _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({})
|
|||
_TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True})
|
||||
|
||||
|
||||
def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[DeploymentTypedDict]:
|
||||
def _deployments_served_for(model: str, llm_router: RegistryReader) -> Sequence[DeploymentTypedDict]:
|
||||
"""
|
||||
The deployments a request to ``model`` routes to, for the gates that waive budget checks.
|
||||
|
||||
|
|
@ -4842,7 +4843,7 @@ async def get_skill_names_from_access_groups(
|
|||
|
||||
def _resolve_all_team_model_sentinel_for_auth_check(
|
||||
models: list[str],
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
team_id: str | None,
|
||||
) -> list[str]:
|
||||
if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
|
||||
|
|
@ -4856,7 +4857,7 @@ def _resolve_all_team_model_sentinel_for_auth_check(
|
|||
|
||||
def check_model_access_helper(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
models: list[str],
|
||||
team_model_aliases: dict[str, str] | None = None,
|
||||
team_id: str | None = None,
|
||||
|
|
@ -4907,7 +4908,7 @@ _check_model_access_helper: Final = check_model_access_helper
|
|||
|
||||
def _owning_team_names_for_team_deployment(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
team_id: str | None,
|
||||
team_model_aliases: Mapping[str, str] | None,
|
||||
) -> tuple[str, ...]:
|
||||
|
|
@ -4925,7 +4926,7 @@ def _owning_team_names_for_team_deployment(
|
|||
|
||||
def _can_object_call_model(
|
||||
model: str | list[str],
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
models: list[str],
|
||||
team_model_aliases: dict[str, str] | None = None,
|
||||
team_id: str | None = None,
|
||||
|
|
@ -5052,7 +5053,7 @@ def _resolve_team_alias(
|
|||
|
||||
|
||||
def _live_team_alias_target(
|
||||
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: Router | None
|
||||
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: RegistryReader | None
|
||||
) -> str:
|
||||
target: Final = team_model_aliases.get(model)
|
||||
if target is None:
|
||||
|
|
@ -5539,7 +5540,7 @@ async def can_key_call_model(
|
|||
model: str | list[str],
|
||||
llm_model_list: Sequence[object] | None,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: litellm.Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
prisma_client: DatabaseClient | None = None,
|
||||
) -> Literal[True]:
|
||||
"""
|
||||
|
|
@ -5618,7 +5619,7 @@ async def can_key_call_resolved_model(
|
|||
model: str,
|
||||
llm_model_list: Sequence[object] | None,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: litellm.Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
|
|
@ -5766,7 +5767,7 @@ def can_org_access_model(
|
|||
async def can_team_access_model(
|
||||
model: str | list[str],
|
||||
team_object: LiteLLM_TeamTable | None,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
team_model_aliases: dict[str, str] | None = None,
|
||||
key_model_aliases: Mapping[str, str] | None = None,
|
||||
prisma_client: DatabaseClient | None = None,
|
||||
|
|
@ -5869,7 +5870,7 @@ async def key_access_group_grants_model(
|
|||
model: str | list[str],
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
team_object: LiteLLM_TeamTable | None,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
key_model_aliases: Mapping[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
|
|
@ -5905,7 +5906,7 @@ _key_access_group_grants_model: Final = key_access_group_grants_model
|
|||
def can_project_access_model(
|
||||
model: str | list[str],
|
||||
project_object: LiteLLM_ProjectTable,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
key_model_aliases: Mapping[str, str] | None = None,
|
||||
) -> Literal[True]:
|
||||
"""
|
||||
|
|
@ -5926,7 +5927,7 @@ def can_project_access_model(
|
|||
def can_customer_access_model(
|
||||
model: str | list[str],
|
||||
end_user_object: LiteLLM_EndUserTable,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
) -> Literal[True]:
|
||||
team_model_aliases: Final = team_model_aliases_for_auth_check(valid_token) if valid_token is not None else None
|
||||
|
|
@ -6089,7 +6090,7 @@ async def _team_grants_personal_key_model(
|
|||
model: str,
|
||||
team_object: LiteLLM_TeamTableCachedObj,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
@ -6127,7 +6128,7 @@ async def _loaded_teams_allow_model(
|
|||
model: str,
|
||||
teams: tuple[LiteLLM_TeamTableCachedObj | None, ...],
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
@ -6393,7 +6394,7 @@ async def can_personal_key_call_model_via_teams(
|
|||
model: str | Sequence[str],
|
||||
user_object: LiteLLM_UserTable | None,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
@ -7185,7 +7186,7 @@ async def check_team_member_model_access(
|
|||
model: str | list[str],
|
||||
team_object: LiteLLM_TeamTable,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
llm_router: RegistryReader | None,
|
||||
prisma_client: Optional["PrismaClient"],
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
_is_model_cost_zero, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
is_model_cost_zero, # pyright: ignore[reportPrivateUsage] # the zero-cost predicate the auth-time budget checks use; no public equivalent
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RegistryReader, RoutingContext
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
||||
|
|
@ -118,7 +118,7 @@ async def is_token_within_budget_for_model(
|
|||
*,
|
||||
model: str,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_router: Router,
|
||||
llm_router: RegistryReader,
|
||||
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
|
|
@ -187,19 +187,19 @@ class RouterFallbackBudgetCheck:
|
|||
is_enforced: Callable[[], bool]
|
||||
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None
|
||||
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
|
||||
if not self.is_enforced():
|
||||
return True
|
||||
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
|
||||
if valid_token is None:
|
||||
return True
|
||||
budget_models: Final = tuple(dict.fromkeys((model, llm_router.registry().resolve(model))))
|
||||
budget_models: Final = tuple(dict.fromkeys((model, ctx.registry().resolve(model))))
|
||||
for budget_model in budget_models:
|
||||
try:
|
||||
if not await is_token_within_budget_for_model(
|
||||
model=budget_model,
|
||||
valid_token=valid_token,
|
||||
llm_router=llm_router,
|
||||
llm_router=ctx,
|
||||
team_member_model_budget_limiter=self.team_member_model_budget_limiter,
|
||||
):
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from pydantic import ValidationError
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RegistryReader, RoutingContext
|
||||
from litellm.search import asearch
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ class _FallbackAccessSettings(LiteLLMBaseModel):
|
|||
enforce_fallback_model_access: bool = True
|
||||
|
||||
|
||||
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
|
||||
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: RegistryReader) -> bool:
|
||||
try:
|
||||
await can_key_call_resolved_model(
|
||||
model=model,
|
||||
|
|
@ -98,7 +98,7 @@ class RouterFallbackAccessCheck:
|
|||
|
||||
is_enforced: Callable[[], bool]
|
||||
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
|
||||
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
|
||||
if valid_token is None:
|
||||
return True
|
||||
|
|
@ -106,7 +106,7 @@ class RouterFallbackAccessCheck:
|
|||
return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token)
|
||||
if not self.is_enforced():
|
||||
return True
|
||||
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
|
||||
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx)
|
||||
|
||||
|
||||
router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings)
|
||||
|
|
|
|||
|
|
@ -14,13 +14,13 @@ from typing import Final, TypeVar
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
|
||||
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
|
||||
from litellm.responses.main import strip_responses_routing_prefix
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.router_api import fallback_chain
|
||||
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
|
||||
|
|
@ -708,7 +708,7 @@ def get_all_fallbacks(
|
|||
|
||||
try:
|
||||
# Use existing function to get fallback model group
|
||||
fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks_config, model_group=model)
|
||||
fallback_model_group, _ = fallback_chain(fallbacks=fallbacks_config, model_group=model)
|
||||
|
||||
if fallback_model_group is None:
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
|||
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.models.end_user import LiteLLM_EndUserTable
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
|
||||
|
|
@ -182,13 +183,7 @@ from litellm.proxy.utils import (
|
|||
from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.router_utils.fallback_event_handlers import (
|
||||
PROVIDER_SCOPED_RESOURCE_KEYS,
|
||||
fallback_lookup_groups,
|
||||
fallbacks_disabled_for_request,
|
||||
get_fallback_model_group_for_lookup_groups,
|
||||
)
|
||||
from litellm.router_api import fallbacks_disabled, references_provider_scoped_resource, request_fallback_chain
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
||||
|
|
@ -811,11 +806,7 @@ async def _check_team_member_model_budget_for_model(
|
|||
model=model_name,
|
||||
)
|
||||
except litellm.BudgetExceededError:
|
||||
if (
|
||||
request_data.get("model") != model_name
|
||||
or llm_router is None
|
||||
or fallbacks_disabled_for_request(kwargs=request_data)
|
||||
):
|
||||
if request_data.get("model") != model_name or llm_router is None or fallbacks_disabled(request_data):
|
||||
raise
|
||||
fallback_succeeded: Final = await _try_team_member_model_budget_fallback(
|
||||
user_id=user_id,
|
||||
|
|
@ -857,9 +848,9 @@ async def _try_team_member_model_budget_fallback(
|
|||
user_api_key_cache: UserApiKeyCache | None = None,
|
||||
proxy_logging_obj: ProxyLogging | None = None,
|
||||
) -> bool:
|
||||
if request.headers.get("content-type", "").lower().startswith("multipart/form-data") or any(
|
||||
request_data.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS
|
||||
):
|
||||
if request.headers.get("content-type", "").lower().startswith(
|
||||
"multipart/form-data"
|
||||
) or references_provider_scoped_resource(request_data):
|
||||
return False
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
|
@ -896,10 +887,7 @@ async def _try_team_member_model_budget_fallback(
|
|||
if router_fallbacks is None:
|
||||
return False
|
||||
|
||||
fallback_models, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=list(router_fallbacks),
|
||||
lookup_groups=fallback_lookup_groups(kwargs=request_data, model_group=model_name),
|
||||
)
|
||||
fallback_models, _ = request_fallback_chain(list(router_fallbacks), request_data, model_name)
|
||||
fallback_targets: Final[Sequence[object]] = fallback_models or ()
|
||||
fallback_candidates: Final[tuple[str, ...]] = tuple(
|
||||
model for target in fallback_targets if (model := _budget_fallback_target_model(target)) is not None
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
build_bug_report,
|
||||
build_environment_report,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_groups import VALID_ROUTING_STRATEGIES
|
||||
from litellm.proxy._types import ConfigGeneralSettings
|
||||
from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES
|
||||
from litellm.types.caching import LiteLLMCacheType
|
||||
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations
|
||||
from litellm.types.secret_managers.main import KeyManagementSystem
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
bug_report_notice,
|
||||
|
|
@ -81,6 +82,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
|
|||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
get_response_headers,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.served_output_texts import (
|
||||
record_served_output_texts,
|
||||
|
|
@ -146,8 +148,6 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
|
|||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
|
@ -2565,7 +2565,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
aliased_model: Final = self.data.get("model")
|
||||
if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
|
||||
self.data.update(
|
||||
llm_router.priority_header_update(
|
||||
llm_router.health().priority_header_fields(
|
||||
model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
|
||||
)
|
||||
)
|
||||
|
|
@ -2831,12 +2831,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
@staticmethod
|
||||
def _resolve_fallback_models(model: str, fallbacks: list) -> list[str] | None:
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
from litellm.router_api import fallback_chain
|
||||
|
||||
fallback_model_group, generic_fallback_idx = get_fallback_model_group(
|
||||
fallbacks=fallbacks,
|
||||
model_group=model,
|
||||
)
|
||||
fallback_model_group, generic_fallback_idx = fallback_chain(fallbacks, model)
|
||||
if fallback_model_group is not None:
|
||||
return fallback_model_group
|
||||
return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None
|
||||
|
|
@ -4190,8 +4187,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
HiddenParamsAsyncIteratorWrapper, so class sniffing runs on the
|
||||
unwrapped inner iterator.
|
||||
"""
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
|
||||
|
||||
unwrapped: Final = response.inner if isinstance(response, HiddenParamsAsyncIteratorWrapper) else response
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_f
|
|||
from litellm.llms.anthropic.cache_aware_routing import AnthropicCacheRouting, TokenCounter
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import Deployment, PreRoutingHookResponse
|
||||
|
|
@ -23,6 +24,7 @@ from litellm.types.utils import StandardLoggingRoutingDecision
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
_MESSAGES: Final = TypeAdapter(list[Mapping[str, object]])
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -231,7 +233,7 @@ async def select_cached_model(
|
|||
|
||||
async def _choose_cached_model(
|
||||
*,
|
||||
router: Router,
|
||||
ctx: RoutingContext,
|
||||
config: ComplexityRouterConfig,
|
||||
params_for_model: Callable[[str, str], Mapping[str, object]],
|
||||
response: PreRoutingHookResponse | None,
|
||||
|
|
@ -249,8 +251,10 @@ async def _choose_cached_model(
|
|||
)
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
|
||||
|
||||
router: Final = proxy_server.llm_router
|
||||
if (
|
||||
proxy_server.llm_router is not router
|
||||
router is None
|
||||
or not ctx.belongs_to(router)
|
||||
or router.registry().plugins("routing")
|
||||
or has_request_transforms()
|
||||
or compaction_pending(request_kwargs)
|
||||
|
|
@ -318,9 +322,10 @@ async def _choose_cached_model(
|
|||
)
|
||||
|
||||
|
||||
@renamed_router_param("router")
|
||||
async def choose_cached_model(
|
||||
*,
|
||||
router: Router,
|
||||
ctx: RoutingContext,
|
||||
config: ComplexityRouterConfig,
|
||||
params_for_model: Callable[[str, str], Mapping[str, object]],
|
||||
response: PreRoutingHookResponse | None,
|
||||
|
|
@ -332,7 +337,7 @@ async def choose_cached_model(
|
|||
try:
|
||||
return await asyncio.wait_for(
|
||||
_choose_cached_model(
|
||||
router=router,
|
||||
ctx=ctx,
|
||||
config=config,
|
||||
params_for_model=params_for_model,
|
||||
response=response,
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ from types import MappingProxyType
|
|||
from typing import Final, cast # noqa: TID251 # narrows untyped credential values at the decrypt boundary
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.clientside_credential_handler import clientside_credential_keys
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.router_utils.clientside_credential_handler import clientside_credential_keys
|
||||
from litellm.types.router import (
|
||||
GenericLiteLLMParams,
|
||||
server_owned_wif_fields_present,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import CLAUDE_CODE_CLIENT, CLAUDE_CODE_PICKER_PATTERN, GATEWAY_CLIENT_HEADER
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
has_tool_with_name,
|
||||
)
|
||||
from litellm.litellm_core_utils.provider_affinity import get_caller_session_id
|
||||
from litellm.litellm_core_utils.routing_common_utils import request_user_api_key_auth
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -40,7 +41,6 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import (
|
|||
merge_rewritten_text_parts,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
|
||||
from litellm.router_utils.common_utils import request_user_api_key_auth
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.guardrails import GuardrailEventHooks, Mode
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
|
@ -134,7 +135,7 @@ class SemanticGuardRouteLoader:
|
|||
)
|
||||
|
||||
encoder: Final = LiteLLMRouterEncoder(
|
||||
litellm_router_instance=litellm_router,
|
||||
ctx=RoutingContext(litellm_router),
|
||||
model_name=embedding_model,
|
||||
score_threshold=global_threshold,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -27,12 +27,12 @@ from litellm.constants import (
|
|||
HEALTH_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependency,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
|
||||
from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
|
||||
|
||||
# Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ from litellm.caching.redis_cache import RedisCache
|
|||
from litellm.constants import (
|
||||
DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL,
|
||||
DEFAULT_SHARED_HEALTH_CHECK_TTL,
|
||||
HEALTH_CHECKS_TARGET,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
from litellm.router_utils.health_state_cache import HEALTH_CHECKS_TARGET
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
|
|||
|
|
@ -26,7 +26,12 @@ from litellm.integrations.SlackAlerting.ms_teams import (
|
|||
build_ms_teams_payload,
|
||||
get_ms_teams_webhook_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.clientside_credential_handler import (
|
||||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
|
||||
clientside_credential_keys,
|
||||
)
|
||||
from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import (
|
||||
AlertType,
|
||||
|
|
@ -94,11 +99,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
|
|||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import deployment_usable_by_team
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
|
||||
clientside_credential_keys,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
|
||||
from litellm.types.router import reject_server_owned_wif_fields
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import (
|
||||
ProxyRateLimitError,
|
||||
|
|
@ -34,10 +38,6 @@ from litellm.proxy.hooks.rate_limiter_utils import (
|
|||
resolve_llm_provider_for_rate_limit,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.types.router import ModelGroupInfo
|
||||
from litellm.types.utils import CallTypesLiteral, LLMResponseTypes
|
||||
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ def _team_public_names_by_group(router: "Router", team_id: str | None) -> Mappin
|
|||
groups_by_name: Final = MappingProxyType(
|
||||
{
|
||||
name: model_access_groups_of(router=router, model=name, team_id=team_id)
|
||||
for name in router.team_public_model_names_of(team_id)
|
||||
for name in router.registry().names(kind="team_public", team_id=team_id)
|
||||
}
|
||||
)
|
||||
return MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -56,6 +60,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
split_media_for_tpm,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
|
@ -86,12 +92,6 @@ from litellm.proxy.spend_tracking.input_tokens import (
|
|||
openai_chat_pdf_file_tokens,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
|
|
@ -3561,9 +3561,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
Returns True if any deployment has failures in the current minute.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
get_deployment_failures_for_current_minute,
|
||||
)
|
||||
|
||||
if llm_router is None:
|
||||
return False
|
||||
|
|
@ -3581,10 +3578,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
continue
|
||||
|
||||
# Use router's existing failure tracking
|
||||
failure_count = get_deployment_failures_for_current_minute(
|
||||
litellm_router_instance=llm_router,
|
||||
deployment_id=deployment_id,
|
||||
)
|
||||
failure_count = llm_router.health().failures_this_minute(deployment_id)
|
||||
|
||||
if failure_count > DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
|
|||
|
|
@ -3291,7 +3291,7 @@ def _fallback_edges(
|
|||
and its provider-stripped suffix, plus the router matcher run over the fuzzy
|
||||
entries that have not fired yet (tracked in ``fired_fuzzy`` by
|
||||
``(list_index, entry_index)``), so each entry's targets are expanded once."""
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
from litellm.router_api import fallback_chain
|
||||
|
||||
stripped: Final = model_group.split("/", 1)[1] if "/" in model_group else None
|
||||
targets: Final[set[str]] = set() # mutable-ok: accumulates fallback targets
|
||||
|
|
@ -3302,7 +3302,7 @@ def _fallback_edges(
|
|||
pending = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy)
|
||||
if not pending:
|
||||
continue
|
||||
resolved = get_fallback_model_group(fallbacks=[fuzzy[index] for index in pending], model_group=model_group)[0]
|
||||
resolved = fallback_chain([fuzzy[index] for index in pending], model_group)[0]
|
||||
values = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else [])
|
||||
targets.update(name for value in values if (name := _fallback_target_name(value)) is not None)
|
||||
if resolved is not None:
|
||||
|
|
@ -3489,7 +3489,7 @@ async def _resolve_user_provider_credentials_for_request(
|
|||
status_code=400,
|
||||
detail=(f"fallback lists in the request body cannot exceed {_FALLBACK_DISCOVERY_LIMIT} entries"),
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
|
||||
# Discovery must see the group the router actually routes to, so include the
|
||||
# post-alias name from every alias map that may still rewrite data["model"]
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.repositories.table_repositories import PrismaTableRepository
|
|||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependencyRole,
|
||||
classify_strategy_router_model,
|
||||
|
|
@ -614,7 +615,7 @@ async def preview_auto_router_routing(
|
|||
try:
|
||||
complexity_strategy: Final = ComplexityRouter(
|
||||
model_name=resolved.router_name,
|
||||
litellm_router_instance=llm_router,
|
||||
ctx=RoutingContext(llm_router),
|
||||
complexity_router_config=resolved.complexity_router_config.model_dump(exclude_none=True),
|
||||
default_model=resolved.complexity_router_config.resolve_default_model(resolved.default_model),
|
||||
derive_savings_baseline=False,
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.ptu_shares import (
|
||||
from litellm.litellm_core_utils.ptu_shares import (
|
||||
model_group_ptu_capacity,
|
||||
routed_deployments,
|
||||
team_servable_deployments,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
DailySpendData,
|
||||
MetricWithMetadata,
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
|
||||
from litellm.litellm_core_utils.routing_common_utils import provider_for_generic_call
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.router_utils.common_utils import provider_for_generic_call
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ from typing import Final
|
|||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.litellm_core_utils.ptu_shares import deployments_shared_with
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.ptu_shares import deployments_shared_with
|
||||
|
||||
|
||||
def raise_if_teams_hold_ptu_shares(team_ids: Sequence[str], llm_router: Router | None) -> None:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
|
|||
from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager
|
||||
from litellm.proxy.mcp_server.faults import iter_exception_tree
|
||||
from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
|
@ -172,7 +173,7 @@ class SemanticMCPToolFilter:
|
|||
self.tool_router = SemanticRouter(
|
||||
routes=routes,
|
||||
encoder=LiteLLMRouterEncoder(
|
||||
litellm_router_instance=self.router_instance,
|
||||
ctx=RoutingContext(self.router_instance),
|
||||
model_name=self.embedding_model,
|
||||
score_threshold=self.similarity_threshold,
|
||||
),
|
||||
|
|
@ -247,7 +248,7 @@ class SemanticMCPToolFilter:
|
|||
router: Final = SemanticRouter(
|
||||
routes=[],
|
||||
encoder=LiteLLMRouterEncoder(
|
||||
litellm_router_instance=self.router_instance,
|
||||
ctx=RoutingContext(self.router_instance),
|
||||
model_name=self.embedding_model,
|
||||
score_threshold=self.similarity_threshold,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1786,7 +1786,7 @@ def _routed_model_name_on_provider(
|
|||
def _stored_model_name_routes_to_provider(
|
||||
llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
|
||||
) -> bool:
|
||||
routing_group: Final = llm_router.get_routing_group(model_name)
|
||||
routing_group: Final = llm_router.registry().routing_group(model_name)
|
||||
routed_model_names: Final = (
|
||||
tuple(routing_group.models) if routing_group is not None else (llm_router.registry().resolve(model_name),)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
add_guardrail_to_applied_guardrails_header,
|
||||
add_policy_sources_to_metadata,
|
||||
|
|
@ -18,7 +19,6 @@ from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher
|
|||
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
|
||||
from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.types.proxy.policy_engine import PolicyMatchContext
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
|
||||
|
||||
|
|
|
|||
|
|
@ -89,6 +89,11 @@ from litellm.constants import (
|
|||
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
)
|
||||
from litellm.litellm_core_utils.access_windows import access_windows_config_error
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
|
|
@ -100,6 +105,8 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
initialize_sentry,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.routing_groups import parse_routing_groups
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
|
|
@ -189,13 +196,6 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
|||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.routing_groups import parse_routing_groups
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
|
|
@ -999,7 +999,7 @@ from litellm.router import (
|
|||
LiteLLM_Params,
|
||||
ModelGroupInfo,
|
||||
)
|
||||
from litellm.router_api import ChangeSet, deployment_id
|
||||
from litellm.router_api import ChangeSet, deployment_id, router_init_args
|
||||
from litellm.scheduler import FlowItem, Scheduler
|
||||
from litellm.secret_managers.aws_secret_manager import load_aws_kms
|
||||
from litellm.secret_managers.google_kms import load_google_kms
|
||||
|
|
@ -4660,10 +4660,6 @@ def _write_health_state_to_router_cache(
|
|||
is on.
|
||||
"""
|
||||
from litellm.proxy.health_check import build_deployment_health_states
|
||||
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
increment_deployment_failures_for_current_minute,
|
||||
)
|
||||
|
||||
_exceptions: Final[dict] = exceptions_by_model_id or {}
|
||||
|
||||
|
|
@ -4715,18 +4711,7 @@ def _write_health_state_to_router_cache(
|
|||
):
|
||||
continue
|
||||
|
||||
increment_deployment_failures_for_current_minute(
|
||||
litellm_router_instance=llm_router,
|
||||
deployment_id=model_id,
|
||||
)
|
||||
|
||||
set_cooldown_deployments(
|
||||
litellm_router_instance=llm_router,
|
||||
original_exception=original_exception,
|
||||
exception_status=exception_status,
|
||||
deployment=model_id,
|
||||
time_to_cooldown=router_settings.cooldown_time,
|
||||
)
|
||||
llm_router.health().record_failure(model_id, original_exception, exception_status)
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning("Failed to write health state to router cache: %s", str(e))
|
||||
|
|
@ -7370,9 +7355,7 @@ class ProxyConfig:
|
|||
router_settings: Final = config.get("router_settings", None)
|
||||
|
||||
if router_settings and isinstance(router_settings, dict):
|
||||
available_args: Final = [
|
||||
x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
|
||||
]
|
||||
available_args: Final = [x for x in router_init_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG]
|
||||
|
||||
for k, v in router_settings.items():
|
||||
if k in available_args:
|
||||
|
|
@ -9596,7 +9579,6 @@ class ProxyConfig:
|
|||
SearchToolRegistry,
|
||||
keep_loaded_search_tools_that_do_not_decrypt,
|
||||
)
|
||||
from litellm.router_utils.search_api_router import SearchAPIRouter
|
||||
|
||||
try:
|
||||
db_search_tools: Final = keep_loaded_search_tools_that_do_not_decrypt(
|
||||
|
|
@ -9620,7 +9602,21 @@ class ProxyConfig:
|
|||
)
|
||||
|
||||
if llm_router is not None:
|
||||
await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools)
|
||||
llm_router.apply(
|
||||
ChangeSet(
|
||||
assign={
|
||||
"search_tools": [
|
||||
{
|
||||
"search_tool_id": tool.get("search_tool_id"),
|
||||
"search_tool_name": tool.get("search_tool_name"),
|
||||
"litellm_params": tool.get("litellm_params", {}),
|
||||
"search_tool_info": tool.get("search_tool_info"),
|
||||
}
|
||||
for tool in search_tools
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
verbose_proxy_logger.info("Successfully loaded %s search tool(s) into router", len(search_tools))
|
||||
else:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -16152,7 +16148,7 @@ def _enrich_model_info_with_litellm_data(
|
|||
if debug is True:
|
||||
_openai_client = "None"
|
||||
if llm_router is not None:
|
||||
_openai_client = llm_router._get_client(deployment=model, kwargs={}, client_type="async") or "None"
|
||||
_openai_client = llm_router.registry().client(model, client_type="async") or "None"
|
||||
else:
|
||||
_openai_client = "llm_router_is_None"
|
||||
openai_client: Final = str(_openai_client)
|
||||
|
|
@ -18602,7 +18598,7 @@ async def async_queue_request(
|
|||
router: Final = llm_router
|
||||
|
||||
data.update(
|
||||
router.priority_header_update(
|
||||
router.health().priority_header_fields(
|
||||
model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
|
||||
)
|
||||
)
|
||||
|
|
@ -19751,7 +19747,7 @@ async def update_config(
|
|||
raw_router_settings: Final = request_body.get("router_settings")
|
||||
if isinstance(raw_router_settings, dict):
|
||||
supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | (
|
||||
frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
|
||||
frozenset(router_init_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
|
||||
)
|
||||
unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings)
|
||||
if unsupported_router_settings:
|
||||
|
|
|
|||
|
|
@ -16,8 +16,9 @@ from pydantic import BeforeValidator, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
from litellm.router_api import router_init_args
|
||||
|
||||
# Client-supplied params that make the router or the call path fabricate a
|
||||
# failure or a delay instead of calling the provider. The ``mock_testing_*``
|
||||
|
|
@ -93,7 +94,7 @@ async def _route_user_config_request(data: dict, route_type: str):
|
|||
|
||||
# Filter router_config to only include valid Router.__init__ arguments
|
||||
# This prevents TypeError when invalid parameters are stored in the database
|
||||
valid_args: Final = litellm.Router.get_valid_args()
|
||||
valid_args: Final = router_init_args()
|
||||
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
|
||||
|
||||
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Final
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
deployment_openai_model,
|
||||
pdf_page_pricing,
|
||||
|
|
@ -31,7 +32,6 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
|
|||
billable_input_since_newest_compaction,
|
||||
holds_compaction_summary,
|
||||
)
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
from litellm.rust_bridge import runtime
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge.token_counter import (
|
||||
|
|
@ -162,7 +162,7 @@ def models_counted_for(model: str, llm_router: Router | None, team_id: str | Non
|
|||
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
|
||||
registry: Final = llm_router.registry()
|
||||
resolved: Final = registry.resolve(model, kind="alias") or model
|
||||
routing_group: Final = llm_router.get_routing_group(resolved)
|
||||
routing_group: Final = registry.routing_group(resolved)
|
||||
members: Final = routing_group.models if routing_group is not None else (resolved,)
|
||||
named: Final = tuple(
|
||||
chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members)
|
||||
|
|
|
|||
|
|
@ -160,10 +160,12 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
independent_snapshot,
|
||||
is_expected_client_error,
|
||||
)
|
||||
from litellm.litellm_core_utils.deployment_pricing import pricing_models
|
||||
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
|
||||
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.served_output_texts import (
|
||||
|
|
@ -278,8 +280,6 @@ from litellm.repositories.user_repository import UserRepository
|
|||
from litellm.repositories.verification_token_repository import (
|
||||
VerificationTokenRepository,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.deployment_pricing import pricing_models
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
|
|
|||
|
|
@ -42,6 +42,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
update_responses_tools_with_model_file_ids,
|
||||
)
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
deployment_reasoning_effort_levels,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
)
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
|
|
@ -63,11 +68,6 @@ from litellm.responses.mcp.round_billing import (
|
|||
)
|
||||
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
deployment_reasoning_effort_levels,
|
||||
model_group_from_metadata,
|
||||
resolve_deployment_reasoning_effort,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
PromptObject,
|
||||
Reasoning,
|
||||
|
|
|
|||
|
|
@ -106,6 +106,20 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
|
||||
from litellm.litellm_core_utils.access_windows import access_windows_config_error, filter_reserved_deployments
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
add_retry_headers_to_response,
|
||||
apply_quality_router_decision_headers,
|
||||
apply_remaining_usage_headers,
|
||||
apply_response_model_id,
|
||||
complexity_router_decision_headers,
|
||||
ensure_response_additional_headers,
|
||||
get_hidden_params_dict,
|
||||
prepare_response_for_header_attachment,
|
||||
replace_complexity_router_headers,
|
||||
response_total_token_count,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
|
|
@ -116,6 +130,10 @@ from litellm.litellm_core_utils.batch_utils import (
|
|||
should_replace_model_in_jsonl,
|
||||
without_deployments_that_cannot_batch,
|
||||
)
|
||||
from litellm.litellm_core_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
is_clientside_credential,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
|
|
@ -127,12 +145,17 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
inferred_provider,
|
||||
is_registered_custom_provider,
|
||||
resolve_model_and_provider_for_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_retry_from_policy import (
|
||||
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_retry_from_policy import resolve_retry_policy
|
||||
from litellm.litellm_core_utils.hidden_params import (
|
||||
HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR,
|
||||
)
|
||||
|
|
@ -142,6 +165,7 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
PTU_COST_ATTRIBUTION_ENV_VAR,
|
||||
declares_ptu,
|
||||
|
|
@ -152,10 +176,52 @@ from litellm.litellm_core_utils.ptu_pricing import (
|
|||
ptu_terms,
|
||||
zeroed_ptu_pricing,
|
||||
)
|
||||
from litellm.litellm_core_utils.ptu_shares import (
|
||||
PTUTeamCeiling,
|
||||
filter_ptu_shared_deployments,
|
||||
ptu_capacity_warning,
|
||||
ptu_share_teams,
|
||||
share_holder_request_kwargs,
|
||||
team_ptu_ceiling,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import (
|
||||
deployment_is_catalog_mapped,
|
||||
intersect_supported_reasoning_efforts,
|
||||
resolve_supported_reasoning_efforts,
|
||||
)
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import (
|
||||
get_configured_request_timeout,
|
||||
get_service_tier_timeout,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
as_retry_skipped_deployment_ids,
|
||||
deployment_model_id_pin,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
format_content_policy_fallback_hint,
|
||||
format_content_policy_no_fallback_note,
|
||||
format_fallback_outcome_message,
|
||||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
provider_for_generic_call,
|
||||
request_user_api_key_auth,
|
||||
resolve_model_group_alias,
|
||||
team_may_use_deployment,
|
||||
truncate_fallback_error_detail,
|
||||
warn_on_ignored_model_capability_flags,
|
||||
warn_on_provider_credential_mismatch,
|
||||
with_admin_api_key_secret_reference,
|
||||
without_model_capability_flags,
|
||||
without_router_only_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_groups import (
|
||||
apply_routing_group_priority,
|
||||
parse_routing_groups,
|
||||
validate_routing_strategy,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import (
|
||||
SensitiveDataMasker,
|
||||
|
|
@ -196,6 +262,7 @@ from litellm.router_api import (
|
|||
aapply_changes,
|
||||
apply_changes,
|
||||
)
|
||||
from litellm.router_api import RoutingContext as RouterContext
|
||||
from litellm.router_api import deployment_id as _deployment_id
|
||||
from litellm.router_deprecated import DeprecatedRouterMembers
|
||||
from litellm.router_host import router_host
|
||||
|
|
@ -233,49 +300,7 @@ from litellm.router_strategy.tag_based_routing import (
|
|||
is_valid_deployment_tag,
|
||||
tags_with_serving_deployment_tags,
|
||||
)
|
||||
from litellm.router_utils.access_windows import access_windows_config_error, filter_reserved_deployments
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
add_retry_headers_to_response,
|
||||
apply_quality_router_decision_headers,
|
||||
apply_remaining_usage_headers,
|
||||
apply_response_model_id,
|
||||
complexity_router_decision_headers,
|
||||
ensure_response_additional_headers,
|
||||
get_hidden_params_dict,
|
||||
prepare_response_for_header_attachment,
|
||||
replace_complexity_router_headers,
|
||||
response_total_token_count,
|
||||
)
|
||||
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
is_clientside_credential,
|
||||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
as_retry_skipped_deployment_ids,
|
||||
deployment_model_id_pin,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
format_content_policy_fallback_hint,
|
||||
format_content_policy_no_fallback_note,
|
||||
format_fallback_outcome_message,
|
||||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
provider_for_generic_call,
|
||||
request_user_api_key_auth,
|
||||
resolve_model_group_alias,
|
||||
team_may_use_deployment,
|
||||
truncate_fallback_error_detail,
|
||||
warn_on_ignored_model_capability_flags,
|
||||
warn_on_provider_credential_mismatch,
|
||||
with_admin_api_key_secret_reference,
|
||||
without_model_capability_flags,
|
||||
without_router_only_kwargs,
|
||||
)
|
||||
from litellm.router_utils.cooldown_cache import CooldownCache
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
DEFAULT_COOLDOWN_TIME_SECONDS,
|
||||
|
|
@ -292,7 +317,6 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
mark_deployment_selection_failure,
|
||||
set_cooldown_deployments,
|
||||
)
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.router_utils.embedding_batch_size_check import (
|
||||
async_log_rejection_before_provider_call,
|
||||
clear_rejected_before_provider_call,
|
||||
|
|
@ -337,10 +361,6 @@ from litellm.router_utils.fallback_event_handlers import (
|
|||
routed_deployment_id,
|
||||
run_async_fallback,
|
||||
)
|
||||
from litellm.router_utils.get_retry_from_policy import (
|
||||
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
|
||||
)
|
||||
from litellm.router_utils.get_retry_from_policy import resolve_retry_policy
|
||||
from litellm.router_utils.handle_error import (
|
||||
async_raise_no_deployment_exception,
|
||||
send_llm_exception_alert,
|
||||
|
|
@ -392,19 +412,6 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
|
|||
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
|
||||
PromptCachingDeploymentCheck,
|
||||
)
|
||||
from litellm.router_utils.ptu_shares import (
|
||||
PTUTeamCeiling,
|
||||
filter_ptu_shared_deployments,
|
||||
ptu_capacity_warning,
|
||||
ptu_share_teams,
|
||||
share_holder_request_kwargs,
|
||||
team_ptu_ceiling,
|
||||
)
|
||||
from litellm.router_utils.reasoning_effort_capability import (
|
||||
deployment_is_catalog_mapped,
|
||||
intersect_supported_reasoning_efforts,
|
||||
resolve_supported_reasoning_efforts,
|
||||
)
|
||||
from litellm.router_utils.request_priority import (
|
||||
InvalidPriority,
|
||||
parse_default_priority,
|
||||
|
|
@ -420,11 +427,6 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
|||
increment_deployment_failures_for_current_minute,
|
||||
increment_deployment_successes_for_current_minute,
|
||||
)
|
||||
from litellm.router_utils.routing_groups import (
|
||||
apply_routing_group_priority,
|
||||
parse_routing_groups,
|
||||
validate_routing_strategy,
|
||||
)
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
|
||||
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
|
||||
from litellm.scheduler import FlowItem, Scheduler
|
||||
|
|
@ -520,8 +522,6 @@ from litellm.utils import (
|
|||
token_counter,
|
||||
)
|
||||
|
||||
from .router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
|
|
@ -1610,6 +1610,7 @@ class Router(DeprecatedRouterMembers):
|
|||
router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
|
||||
```
|
||||
"""
|
||||
self._core_ctx: Final = RouterContext(self)
|
||||
|
||||
self.set_verbose = set_verbose
|
||||
self.ignore_invalid_deployments = ignore_invalid_deployments
|
||||
|
|
@ -1885,7 +1886,7 @@ class Router(DeprecatedRouterMembers):
|
|||
if self._normalize_strategy(routing_strategy) == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, routing_strategy_args)
|
||||
apply_lar1_routing_strategy(self._core_ctx, routing_strategy_args)
|
||||
else:
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=routing_strategy,
|
||||
|
|
@ -2259,7 +2260,7 @@ class Router(DeprecatedRouterMembers):
|
|||
if strategy == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, self.routing_strategy_args)
|
||||
apply_lar1_routing_strategy(self._core_ctx, self.routing_strategy_args)
|
||||
return
|
||||
|
||||
attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "")
|
||||
|
|
@ -2584,7 +2585,7 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
deployments: Final = self._core_get_model_list(model_name=model)
|
||||
if deployments:
|
||||
RoutingPrefetch.arm(self, usage_selector, deployments)
|
||||
RoutingPrefetch.arm(self._core_ctx, usage_selector, deployments)
|
||||
except Exception as e: # noqa: BLE001 # a prefetch is an optimisation, never a reason to fail the request
|
||||
verbose_router_logger.debug(
|
||||
"routing read prefetch not armed for %s: %s", _without_line_breaks(model), _without_line_breaks(e)
|
||||
|
|
@ -3133,7 +3134,7 @@ class Router(DeprecatedRouterMembers):
|
|||
break
|
||||
|
||||
if existing_ec_callback is not None:
|
||||
existing_ec_callback.router = self
|
||||
existing_ec_callback.ctx = self._core_ctx
|
||||
existing_ec_callback.enable_global_affinity = (
|
||||
existing_ec_callback.enable_global_affinity or enable_global_affinity
|
||||
)
|
||||
|
|
@ -3141,7 +3142,7 @@ class Router(DeprecatedRouterMembers):
|
|||
ec_callback = existing_ec_callback
|
||||
else:
|
||||
ec_callback = EncryptedContentAffinityCheck(
|
||||
router=self,
|
||||
ctx=self._core_ctx,
|
||||
enable_global_affinity=enable_global_affinity,
|
||||
model_group_affinity_config=self.model_group_affinity_config,
|
||||
)
|
||||
|
|
@ -3690,7 +3691,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -4933,7 +4934,7 @@ class Router(DeprecatedRouterMembers):
|
|||
kwargs=kwargs,
|
||||
client_type="max_parallel_requests",
|
||||
)
|
||||
compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat")
|
||||
compacted_input: Final = await compact_to_fit(self._core_ctx, deployment, input_kwargs, "chat")
|
||||
async with contextlib.AsyncExitStack() as deployment_slot:
|
||||
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
|
||||
deployment_slot.enter_context(max_parallel_requests_limit)
|
||||
|
|
@ -5396,7 +5397,7 @@ class Router(DeprecatedRouterMembers):
|
|||
hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs)
|
||||
if _is_fallback_hop(hop_kwargs):
|
||||
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
|
||||
self,
|
||||
self._core_ctx,
|
||||
hop_kwargs.get("input"),
|
||||
hop_kwargs.get("messages"),
|
||||
(_FALLBACK_HOP_ADAPTER.validate_python(deployment),),
|
||||
|
|
@ -6002,7 +6003,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6114,7 +6115,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6207,7 +6208,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6269,7 +6270,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6385,7 +6386,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6455,7 +6456,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6514,7 +6515,7 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
return _OPAQUE_VALUE_ADAPTER.validate_python(
|
||||
await SearchAPIRouter.async_search_with_fallbacks(
|
||||
router_instance=self,
|
||||
litellm_routing_ctx=self._core_ctx,
|
||||
original_function=original_function,
|
||||
**kwargs,
|
||||
)
|
||||
|
|
@ -6528,7 +6529,7 @@ class Router(DeprecatedRouterMembers):
|
|||
from litellm.router_utils.search_api_router import SearchAPIRouter
|
||||
|
||||
return await SearchAPIRouter.async_search_with_fallbacks_helper(
|
||||
router_instance=self,
|
||||
litellm_routing_ctx=self._core_ctx,
|
||||
model=model,
|
||||
original_generic_function=original_generic_function,
|
||||
**kwargs,
|
||||
|
|
@ -6646,7 +6647,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -6746,7 +6747,7 @@ class Router(DeprecatedRouterMembers):
|
|||
response_kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
compacted_input: Final = await compact_to_fit(
|
||||
self,
|
||||
self._core_ctx,
|
||||
deployment,
|
||||
response_kwargs,
|
||||
surface_for_call(generic_function_name),
|
||||
|
|
@ -7624,7 +7625,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -7703,7 +7704,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -7717,7 +7718,7 @@ class Router(DeprecatedRouterMembers):
|
|||
**kwargs,
|
||||
) -> OpenAIFileObject:
|
||||
try:
|
||||
from litellm.router_utils.common_utils import add_model_file_id_mappings
|
||||
from litellm.litellm_core_utils.routing_common_utils import add_model_file_id_mappings
|
||||
|
||||
verbose_router_logger.debug("Inside _atext_completion()- model: %s", model)
|
||||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
|
|
@ -7943,7 +7944,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -8111,7 +8112,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -8142,7 +8143,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception as e:
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -8932,7 +8933,7 @@ class Router(DeprecatedRouterMembers):
|
|||
# that fails with RouterRateLimitError whenever the "remaining" entries
|
||||
# are all in cooldown — the inner async_get_healthy_deployments call
|
||||
# would find an empty list and raise immediately.
|
||||
cooldown_ids = set(await async_get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=None))
|
||||
cooldown_ids = set(await async_get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=None))
|
||||
remaining: Final = (all_ids - cooldown_ids) - excluded
|
||||
if not remaining:
|
||||
return None
|
||||
|
|
@ -9012,7 +9013,7 @@ class Router(DeprecatedRouterMembers):
|
|||
raise e
|
||||
|
||||
input_kwargs: Final = {
|
||||
"litellm_router": self,
|
||||
"litellm_routing_ctx": self._core_ctx,
|
||||
"original_exception": original_exception,
|
||||
**kwargs,
|
||||
"attempted_targets": fallback_walk,
|
||||
|
|
@ -9228,7 +9229,7 @@ class Router(DeprecatedRouterMembers):
|
|||
parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs)
|
||||
fallback_failure_exception_str = truncate_fallback_error_detail(redact_string(str(new_exception)))
|
||||
cooldown_info: Final = await async_get_cooldown_deployments_with_debug_info(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
verbose_router_logger.error(
|
||||
|
|
@ -9849,7 +9850,7 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
# Always track deployment successes for cooldown logic, regardless of TPM/RPM limits
|
||||
increment_deployment_successes_for_current_minute(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
deployment_id=str(deployment_id),
|
||||
)
|
||||
|
||||
|
|
@ -10029,7 +10030,7 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
if id is not None:
|
||||
key: Final = increment_deployment_successes_for_current_minute(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
deployment_id=id,
|
||||
)
|
||||
return key
|
||||
|
|
@ -10140,11 +10141,11 @@ class Router(DeprecatedRouterMembers):
|
|||
if deployment_id is None:
|
||||
return False
|
||||
increment_deployment_failures_for_current_minute(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
deployment_id=deployment_id,
|
||||
)
|
||||
result: Final = set_cooldown_deployments(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
exception_status=exception_status,
|
||||
original_exception=exception,
|
||||
deployment=deployment_id,
|
||||
|
|
@ -10450,9 +10451,7 @@ class Router(DeprecatedRouterMembers):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
unhealthy_deployments: Final = get_cooldown_deployments(
|
||||
litellm_router_instance=self, parent_otel_span=parent_otel_span
|
||||
)
|
||||
unhealthy_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
unhealthy_set: Final = set(unhealthy_deployments)
|
||||
healthy_deployments: list = [d for d in _all_deployments if d["model_info"]["id"] not in unhealthy_set]
|
||||
healthy_deployments = self._filter_blocked_deployments(healthy_deployments)
|
||||
|
|
@ -10480,7 +10479,7 @@ class Router(DeprecatedRouterMembers):
|
|||
pass
|
||||
|
||||
unhealthy_deployments: Final = await async_get_cooldown_deployments(
|
||||
litellm_router_instance=self, parent_otel_span=parent_otel_span
|
||||
ctx=self._core_ctx, parent_otel_span=parent_otel_span
|
||||
)
|
||||
# Convert to set for O(1) lookup instead of O(n)
|
||||
unhealthy_deployments_set: Final = set(unhealthy_deployments)
|
||||
|
|
@ -10538,7 +10537,7 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
)
|
||||
set_cooldown_deployments(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
exception_status=e.status_code,
|
||||
original_exception=e,
|
||||
deployment=deployment["model_info"]["id"],
|
||||
|
|
@ -10622,7 +10621,7 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
if selected is None:
|
||||
no_deployment: Final = await async_raise_no_deployment_exception(
|
||||
litellm_router_instance=self, model=model_group, parent_otel_span=None
|
||||
ctx=self._core_ctx, model=model_group, parent_otel_span=None
|
||||
)
|
||||
raise litellm.RateLimitError(
|
||||
message=str(no_deployment),
|
||||
|
|
@ -11053,7 +11052,7 @@ class Router(DeprecatedRouterMembers):
|
|||
auto_router_config=auto_router_config,
|
||||
default_model=default_model,
|
||||
embedding_model=embedding_model,
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
max_input_chars=(max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS),
|
||||
)
|
||||
self._register_pre_routing_strategy(
|
||||
|
|
@ -11134,7 +11133,7 @@ class Router(DeprecatedRouterMembers):
|
|||
complexity_router: Final[ComplexityRouter] = ComplexityRouter(
|
||||
model_name=deployment.model_name,
|
||||
default_model=default_model,
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
complexity_router_config=complexity_router_config,
|
||||
)
|
||||
self._register_pre_routing_strategy(
|
||||
|
|
@ -11434,7 +11433,7 @@ class Router(DeprecatedRouterMembers):
|
|||
quality_router: Final[QualityRouter] = QualityRouter(
|
||||
model_name=deployment.model_name,
|
||||
default_model=default_model,
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
quality_router_config=quality_router_config,
|
||||
)
|
||||
self._register_pre_routing_strategy(
|
||||
|
|
@ -13655,7 +13654,7 @@ class Router(DeprecatedRouterMembers):
|
|||
while remaining:
|
||||
try:
|
||||
picked = await get_deployments_for_tag(
|
||||
llm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
model=model,
|
||||
healthy_deployments=remaining,
|
||||
request_kwargs=tag_request,
|
||||
|
|
@ -14538,7 +14537,7 @@ class Router(DeprecatedRouterMembers):
|
|||
if routing_strategy == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, kwargs.get("routing_strategy_args"))
|
||||
apply_lar1_routing_strategy(self._core_ctx, kwargs.get("routing_strategy_args"))
|
||||
return
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=routing_strategy, routing_strategy_args=kwargs.get("routing_strategy_args", {})
|
||||
|
|
@ -14631,7 +14630,7 @@ class Router(DeprecatedRouterMembers):
|
|||
cache_key = f"{model_id}_max_parallel_requests_client"
|
||||
client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span)
|
||||
if client is None:
|
||||
InitalizeCachedClient.set_max_parallel_requests_client(litellm_router_instance=self, model=deployment)
|
||||
InitalizeCachedClient.set_max_parallel_requests_client(ctx=self._core_ctx, model=deployment)
|
||||
client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span)
|
||||
return client
|
||||
elif client_type == "async":
|
||||
|
|
@ -15562,10 +15561,10 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
routing_read_batch: Final = RoutingReadBatch.active()
|
||||
cooldown_deployments: Final = (
|
||||
await async_get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
|
||||
await async_get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
if routing_read_batch is None
|
||||
else await routing_read_batch.async_get_cooldown_deployments(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
healthy_deployments=healthy_deployments,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
|
@ -15620,7 +15619,7 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
# check if user wants to do tag based routing
|
||||
healthy_deployments = await get_deployments_for_tag(
|
||||
llm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
healthy_deployments=healthy_deployments,
|
||||
|
|
@ -15699,7 +15698,7 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
if len(healthy_deployments) == 0:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
model=model,
|
||||
parent_otel_span=parent_otel_span,
|
||||
route_model_ids=await self._async_routable_cooled_deployment_ids(
|
||||
|
|
@ -15945,7 +15944,7 @@ class Router(DeprecatedRouterMembers):
|
|||
)
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
model=model,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
|
@ -16115,7 +16114,7 @@ class Router(DeprecatedRouterMembers):
|
|||
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
litellm_router_instance=self,
|
||||
ctx=self._core_ctx,
|
||||
model=model,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
|
@ -16483,7 +16482,7 @@ class Router(DeprecatedRouterMembers):
|
|||
if isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
else (),
|
||||
parent_model=model,
|
||||
router=self,
|
||||
ctx=self._core_ctx,
|
||||
allow_escalation=isinstance(selected_strategy.strategy, ComplexityRouter)
|
||||
and selected_strategy.strategy.config.enable_context_window_escalation,
|
||||
messages=messages,
|
||||
|
|
@ -16846,9 +16845,7 @@ class Router(DeprecatedRouterMembers):
|
|||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
cooldown_deployments: Final = get_cooldown_deployments(
|
||||
litellm_router_instance=self, parent_otel_span=parent_otel_span
|
||||
)
|
||||
cooldown_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
_pre_cooldown_deployments: Final = healthy_deployments
|
||||
healthy_deployments = self._filter_cooldown_deployments(
|
||||
healthy_deployments=healthy_deployments,
|
||||
|
|
@ -16947,7 +16944,7 @@ class Router(DeprecatedRouterMembers):
|
|||
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
|
||||
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
raise RouterRateLimitError(
|
||||
model=model,
|
||||
cooldown_time=_cooldown_time,
|
||||
|
|
@ -16955,7 +16952,7 @@ class Router(DeprecatedRouterMembers):
|
|||
cooldown_list=_cooldown_list,
|
||||
model_ids=model_ids,
|
||||
cooldown_deployments=get_cooldown_deployment_info(
|
||||
self,
|
||||
self._core_ctx,
|
||||
self._deployment_ids(
|
||||
self._region_routable_deployments(
|
||||
request_deployments,
|
||||
|
|
@ -16994,7 +16991,7 @@ class Router(DeprecatedRouterMembers):
|
|||
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
|
||||
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
raise RouterRateLimitError(
|
||||
model=model,
|
||||
cooldown_time=_cooldown_time,
|
||||
|
|
@ -17101,9 +17098,7 @@ class Router(DeprecatedRouterMembers):
|
|||
healthy_deployments=pass_through_deployments,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
cooldown_deployments: Final = get_cooldown_deployments(
|
||||
litellm_router_instance=self, parent_otel_span=parent_otel_span
|
||||
)
|
||||
cooldown_deployments: Final = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
pass_through_deployments = self._filter_cooldown_deployments(
|
||||
healthy_deployments=pass_through_deployments,
|
||||
cooldown_deployments=cooldown_deployments,
|
||||
|
|
@ -17132,7 +17127,7 @@ class Router(DeprecatedRouterMembers):
|
|||
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
|
||||
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
raise RouterRateLimitError(
|
||||
model=model,
|
||||
cooldown_time=_cooldown_time,
|
||||
|
|
@ -17140,7 +17135,7 @@ class Router(DeprecatedRouterMembers):
|
|||
cooldown_list=_cooldown_list,
|
||||
model_ids=pass_through_model_ids,
|
||||
cooldown_deployments=get_cooldown_deployment_info(
|
||||
self,
|
||||
self._core_ctx,
|
||||
self._deployment_ids(
|
||||
self._region_routable_deployments(
|
||||
request_deployments,
|
||||
|
|
@ -17178,7 +17173,7 @@ class Router(DeprecatedRouterMembers):
|
|||
_cooldown_time = self._core_cooldown_cache.get_min_cooldown(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
_cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span)
|
||||
_cooldown_list = get_cooldown_deployments(ctx=self._core_ctx, parent_otel_span=parent_otel_span)
|
||||
raise RouterRateLimitError(
|
||||
model=model,
|
||||
cooldown_time=_cooldown_time,
|
||||
|
|
|
|||
|
|
@ -33,19 +33,28 @@ as they were.
|
|||
# repeat them unchanged, and typing the members themselves is the core's job.
|
||||
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import warnings
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type
|
||||
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.caching.caching import DualCache, RedisCache
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.messages.dispatch import MessagesResult
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
|
||||
from litellm.router_strategy.auto_router.auto_router import AutoRouter
|
||||
|
|
@ -53,12 +62,14 @@ if TYPE_CHECKING:
|
|||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.quality_router.quality_router import QualityRouter
|
||||
from litellm.router_utils.cooldown_cache import CooldownCache
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets
|
||||
from litellm.router_utils.health_state_cache import DeploymentHealthCache
|
||||
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.router import (
|
||||
AllowedFailsPolicy,
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
|
|
@ -74,7 +85,7 @@ if TYPE_CHECKING:
|
|||
SearchToolTypedDict,
|
||||
TaggedPreRoutingStrategy,
|
||||
)
|
||||
from litellm.types.utils import BudgetConfig
|
||||
from litellm.types.utils import BudgetConfig, EmbeddingResponse, ModelResponse
|
||||
from litellm.types.utils import ModelInfo as ModelMapInfo
|
||||
|
||||
# The cores' untyped containers (bare ``list`` / ``dict``, element types Unknown), spelled with Any:
|
||||
|
|
@ -82,6 +93,9 @@ if TYPE_CHECKING:
|
|||
UntypedList: TypeAlias = list[Any] # pyright: ignore[reportExplicitAny] # see above
|
||||
UntypedDict: TypeAlias = dict[Any, Any] # pyright: ignore[reportExplicitAny] # see above
|
||||
|
||||
# Re-exported for litellm.types, which may not import router_utils.
|
||||
AttemptedFallbackTargets: TypeAlias = _AttemptedFallbackTargets
|
||||
|
||||
# What registry().plugins(kind) returns, one member per kind.
|
||||
RegistryPlugins: TypeAlias = (
|
||||
dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]]
|
||||
|
|
@ -184,6 +198,73 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
|
|||
return hashlib.sha256("".join(parts).encode()).hexdigest()
|
||||
|
||||
|
||||
def router_init_args() -> tuple[str, ...]:
|
||||
"""The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
|
||||
from litellm.router import Router
|
||||
|
||||
return tuple(Router.get_valid_args())
|
||||
|
||||
|
||||
def mark_advisor_orchestration_failure(exception: BaseException) -> None:
|
||||
"""Tag an advisor orchestration failure, so the router does not cool down the deployment it picked."""
|
||||
from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure as mark
|
||||
|
||||
mark(exception)
|
||||
|
||||
|
||||
def fallback_chain(
|
||||
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: the fallbacks config shape
|
||||
model_group: str,
|
||||
) -> tuple[list[str] | None, int | None]: # mutable-ok: get_fallback_model_group's type, unchanged
|
||||
"""
|
||||
The fallbacks configured for a model group, and the index of the generic ``*`` entry (was
|
||||
``get_fallback_model_group``). Needs no router instance.
|
||||
"""
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
|
||||
return get_fallback_model_group(fallbacks=fallbacks, model_group=model_group)
|
||||
|
||||
|
||||
def request_fallback_chain(
|
||||
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: the fallbacks config shape
|
||||
request_kwargs: Mapping[str, object],
|
||||
model_group: str | None,
|
||||
) -> tuple[list[str] | None, int | None]: # mutable-ok: get_fallback_model_group's type, unchanged
|
||||
"""
|
||||
``fallback_chain`` for a request: the group a pre-routing hook picked first, then the routed,
|
||||
requested and original groups (was ``get_fallback_model_group_for_lookup_groups``).
|
||||
"""
|
||||
from litellm.router_utils.fallback_event_handlers import (
|
||||
fallback_lookup_groups,
|
||||
get_fallback_model_group_for_lookup_groups,
|
||||
)
|
||||
|
||||
return get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs=request_kwargs, model_group=model_group)
|
||||
)
|
||||
|
||||
|
||||
def fallbacks_disabled(request_kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether the request opted out of fallbacks (was ``fallbacks_disabled_for_request``)."""
|
||||
from litellm.router_utils.fallback_event_handlers import fallbacks_disabled_for_request
|
||||
|
||||
return fallbacks_disabled_for_request(kwargs=request_kwargs)
|
||||
|
||||
|
||||
def references_provider_scoped_resource(request_kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether the request names a file, batch or fine-tuning job that only its own provider knows."""
|
||||
from litellm.router_utils.fallback_event_handlers import PROVIDER_SCOPED_RESOURCE_KEYS
|
||||
|
||||
return any(request_kwargs.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS)
|
||||
|
||||
|
||||
def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
"""The metadata bucket the router stamped the picked deployment into (was ``find_deployment_metadata``)."""
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
|
||||
|
||||
return find_deployment_metadata(request_kwargs)
|
||||
|
||||
|
||||
def _json_default_stable_id(obj: object) -> str:
|
||||
from litellm.router import Router
|
||||
|
||||
|
|
@ -550,7 +631,8 @@ class RouterRegistry:
|
|||
- ``group_set``: the set of configured group names (was the ``model_names`` attribute)
|
||||
- ``ids``: deployment ids, of one group or all (was ``get_model_ids``)
|
||||
- ``deployment_names``: (was the ``deployment_names`` attribute)
|
||||
- ``team_public``: team public model names (was ``team_public_model_names``)
|
||||
- ``team_public``: team public model names (was ``team_public_model_names``), or only the
|
||||
ones ``team_id`` serves its own deployments under (was ``team_public_model_names_of``)
|
||||
- ``litellm_models``: upstream model names a group resolves to (was ``resolved_litellm_models``)
|
||||
- ``litellm_base_models``: the same, with each deployment's ``base_model`` where it sets one
|
||||
(was ``resolved_litellm_models(..., use_base_model=True)``)
|
||||
|
|
@ -568,7 +650,7 @@ class RouterRegistry:
|
|||
if kind == "deployment_names":
|
||||
return r._core_deployment_names
|
||||
if kind == "team_public":
|
||||
return r._core_team_public_model_names
|
||||
return r._core_team_public_model_names if team_id is None else r.team_public_model_names_of(team_id)
|
||||
if model_name is None:
|
||||
raise TypeError(f"registry().names(kind={kind!r}) needs a model_name")
|
||||
if kind == "litellm_models":
|
||||
|
|
@ -832,6 +914,14 @@ class RouterRegistry:
|
|||
"""Routing groups by name (was the ``routing_groups`` attribute)."""
|
||||
return self._router._core_routing_groups
|
||||
|
||||
def routing_group(self, name: str) -> "RoutingGroup | None":
|
||||
"""The routing group callable as ``name``, or None (was ``get_routing_group``)."""
|
||||
return self._router.get_routing_group(name)
|
||||
|
||||
def client(self, deployment: Mapping[str, object], *, client_type: str | None = None) -> object:
|
||||
"""The SDK client the router cached for a deployment, or None (was ``_get_client``)."""
|
||||
return self._router._get_client(deployment=deployment, kwargs={}, client_type=client_type)
|
||||
|
||||
def access_groups(
|
||||
self,
|
||||
model_name: str | None = None,
|
||||
|
|
@ -1128,6 +1218,41 @@ class RouterHealth:
|
|||
"""The request priority and the kwargs without it (was ``split_request_priority``)."""
|
||||
return self._router._core_split_request_priority(model, kwargs)
|
||||
|
||||
def failures_this_minute(self, deployment_id: str) -> int:
|
||||
"""Failures counted against a deployment this minute (was ``get_deployment_failures_for_current_minute``)."""
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
get_deployment_failures_for_current_minute,
|
||||
)
|
||||
|
||||
return get_deployment_failures_for_current_minute(self._router._core_ctx, deployment_id)
|
||||
|
||||
def record_failure(self, deployment_id: str, exception: Exception, exception_status: str | int) -> bool:
|
||||
"""
|
||||
Count a failure against a deployment this minute and cool it down once it is over its allowed
|
||||
fails (was ``increment_deployment_failures_for_current_minute`` then ``set_cooldown_deployments``).
|
||||
Returns whether the deployment was cooled down.
|
||||
"""
|
||||
from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
increment_deployment_failures_for_current_minute,
|
||||
)
|
||||
|
||||
ctx: Final = self._router._core_ctx
|
||||
increment_deployment_failures_for_current_minute(ctx, deployment_id)
|
||||
return set_cooldown_deployments(
|
||||
ctx,
|
||||
original_exception=exception,
|
||||
exception_status=exception_status,
|
||||
deployment=deployment_id,
|
||||
time_to_cooldown=self._router._core_cooldown_time,
|
||||
)
|
||||
|
||||
def priority_header_fields(
|
||||
self, model: str, kwargs: Mapping[str, object], header: str | None
|
||||
) -> Mapping[str, int | str]:
|
||||
"""The request fields a priority header sets (was ``priority_header_update``)."""
|
||||
return self._router.priority_header_update(model=model, kwargs=kwargs, header=header)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChangeSet:
|
||||
|
|
@ -1202,6 +1327,40 @@ async def aapply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
|
|||
return result
|
||||
|
||||
|
||||
def _takes_legacy_llm_router(check: object) -> bool:
|
||||
"""Whether a fallback check still declares ``llm_router`` instead of ``ctx`` (the pre-RoutingContext shape)."""
|
||||
try:
|
||||
params: Final = inspect.signature(check).parameters # pyright: ignore[reportArgumentType] # any callable
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return "ctx" not in params and "llm_router" in params
|
||||
|
||||
|
||||
async def _run_fallback_check(
|
||||
check: "FallbackAccessCheck | FallbackBudgetCheck",
|
||||
ctx: "RoutingContext",
|
||||
*,
|
||||
model: str,
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
if _takes_legacy_llm_router(check):
|
||||
warnings.warn(
|
||||
f"{type(check).__name__} takes llm_router=; fallback checks now receive ctx: RoutingContext. "
|
||||
"Passing llm_router is deprecated and will be removed",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
legacy: Final = check # pyright: ignore[reportAny] # the old protocol shape, checked above
|
||||
return await legacy(model=model, request_kwargs=request_kwargs, llm_router=ctx._router) # pyright: ignore[reportCallIssue, reportPrivateUsage] # see above
|
||||
return await check(model=model, request_kwargs=request_kwargs, ctx=ctx)
|
||||
|
||||
|
||||
class RegistryReader(Protocol):
|
||||
"""Anything that answers ``registry()``: a Router or a RoutingContext. What proxy auth helpers read."""
|
||||
|
||||
def registry(self) -> "RouterRegistry": ...
|
||||
|
||||
|
||||
class RoutingContext:
|
||||
"""
|
||||
What a routing plugin (strategy, pre-call check, sub-router) may use of the router. Plugins get
|
||||
|
|
@ -1225,6 +1384,98 @@ class RoutingContext:
|
|||
def health(self) -> RouterHealth:
|
||||
return self._router.health()
|
||||
|
||||
def observers(self) -> RoutingObservers:
|
||||
return self._router.observe()
|
||||
|
||||
def belongs_to(self, router: "Router") -> bool:
|
||||
return self._router is router
|
||||
|
||||
@property
|
||||
def explicit_allowed_fails(self) -> int | None:
|
||||
"""``allowed_fails`` as the caller passed it, ``None`` when it came from the global default."""
|
||||
return self._router.explicit_allowed_fails
|
||||
|
||||
async def select(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict
|
||||
*,
|
||||
kind: SelectKind = "pick",
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list
|
||||
input: str | list[object] | None = None, # mutable-ok: the selection cores take a list
|
||||
specific_deployment: bool | None = False,
|
||||
parent_otel_span: "Span | None" = None,
|
||||
health_check_probe: bool = False,
|
||||
) -> object:
|
||||
return await self._router.select( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportCallIssue, reportArgumentType] # the selection cores return untyped deployment dicts
|
||||
model,
|
||||
request_kwargs,
|
||||
kind=kind, # pyright: ignore[reportArgumentType] # forwarded as given, the overloads narrow it
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
parent_otel_span=parent_otel_span,
|
||||
health_check_probe=health_check_probe,
|
||||
)
|
||||
|
||||
async def with_fallbacks(
|
||||
self,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: the router's fallback loop takes the request's own keyword payload
|
||||
) -> object:
|
||||
"""Run a call through the router's retry and fallback loop (was ``async_function_with_fallbacks``)."""
|
||||
return await self._router.async_function_with_fallbacks(*args, **kwargs)
|
||||
|
||||
async def acompletion(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "ModelResponse | CustomStreamWrapper":
|
||||
return await self._router.acompletion(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
|
||||
|
||||
async def aresponses(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "ResponsesAPIResponse | BaseResponsesAPIStreamingIterator":
|
||||
return await self._router.aresponses(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.aresponses returns
|
||||
|
||||
async def aembedding(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "EmbeddingResponse":
|
||||
return await self._router.aembedding(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
|
||||
|
||||
def embedding(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "EmbeddingResponse":
|
||||
return self._router.embedding(**kwargs) # pyright: ignore[reportCallIssue, reportArgumentType] # forwarded as given
|
||||
|
||||
async def adecisions(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "OpenAIDecisionResponse | DecisionsResponse":
|
||||
return await self._router.adecisions(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.adecisions returns
|
||||
|
||||
async def aanthropic_messages(
|
||||
self,
|
||||
**kwargs: object, # kwargs-ok: forwarded to the router endpoint as given
|
||||
) -> "MessagesResult":
|
||||
return await self._router.aanthropic_messages(**kwargs) # pyright: ignore[reportReturnType, reportArgumentType] # factory-built endpoint, returns what litellm.anthropic_messages returns
|
||||
|
||||
def apply(self, changes: ChangeSet) -> ApplyResult:
|
||||
return self._router.apply(changes)
|
||||
|
||||
def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None:
|
||||
self._router.register_plugin(plugin)
|
||||
|
||||
async def fallback_access_allowed(self, *, model: str, request_kwargs: Mapping[str, object]) -> bool:
|
||||
check: Final = self._router.settings().fallback_access_check
|
||||
return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs)
|
||||
|
||||
async def fallback_budget_allowed(self, *, model: str, request_kwargs: Mapping[str, object]) -> bool:
|
||||
check: Final = self._router.settings().fallback_budget_check
|
||||
return check is None or await _run_fallback_check(check, self, model=model, request_kwargs=request_kwargs)
|
||||
|
||||
def pre_call_checks(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1292,7 +1543,7 @@ class RoutingContext:
|
|||
self,
|
||||
kwargs: dict[str, object], # mutable-ok: Router._core_log_retry records the retry into the request kwargs
|
||||
e: Exception,
|
||||
) -> Mapping[str, object]:
|
||||
) -> dict: # mutable-ok: the fallback loop edits it; Router._core_log_retry's type
|
||||
return self._router._core_log_retry(kwargs=kwargs, e=e)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,11 +40,11 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm.caching.caching import RedisCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.router import (
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
|
|
@ -838,6 +838,109 @@ class DeprecatedRouterMembers:
|
|||
return _as_router(self)._core_log_retry(kwargs=kwargs, e=e)
|
||||
|
||||
|
||||
# Router-family functions and classes that took the Router (``litellm_router_instance=``,
|
||||
# ``router=``, ...) take its RoutingContext now. These keep the old spellings working.
|
||||
|
||||
_RoutingCallable: Final = TypeVar("_RoutingCallable", bound=Callable[..., object])
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _router_class() -> "type[Router]":
|
||||
from litellm.router import Router
|
||||
|
||||
return Router
|
||||
|
||||
|
||||
def _note_renamed(label: str, old: str, instead: str, *, stacklevel: int = 3) -> None:
|
||||
DEPRECATED_ROUTER_MEMBER_USES[label] += 1
|
||||
warnings.warn(
|
||||
f"{label}: {old} is deprecated; {instead} (the Router's RoutingContext) instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=stacklevel, # the caller of the wrapper or descriptor that noted it
|
||||
)
|
||||
|
||||
|
||||
def _routing_context_of(value: object) -> object:
|
||||
"""The Router's RoutingContext when ``value`` is a Router, else ``value`` as given."""
|
||||
return value._core_ctx if isinstance(value, _router_class()) else value
|
||||
|
||||
|
||||
def renamed_router_param(old: str, new: str = "ctx") -> Callable[[_RoutingCallable], _RoutingCallable]:
|
||||
"""
|
||||
Keep a router parameter that became a RoutingContext (``new``) callable the old way: ``old=``
|
||||
still works, and so does a Router passed where ``new`` goes. Either gets the Router's
|
||||
RoutingContext and a DeprecationWarning that names ``new``.
|
||||
"""
|
||||
|
||||
def decorate(fn: _RoutingCallable) -> _RoutingCallable:
|
||||
index: Final = list(inspect.signature(fn).parameters).index(new)
|
||||
label: Final = f"{fn.__module__}.{fn.__qualname__}"
|
||||
|
||||
def convert(
|
||||
args: tuple[object, ...],
|
||||
kwargs: dict[str, object], # mutable-ok: the call's own kwargs
|
||||
) -> tuple[object, ...]:
|
||||
if old in kwargs:
|
||||
if new in kwargs:
|
||||
raise TypeError(f"{fn.__qualname__}() got both {old}= and {new}=")
|
||||
_note_renamed(label, f"{old}=", f"pass {new}=", stacklevel=4)
|
||||
kwargs[new] = _routing_context_of(kwargs.pop(old))
|
||||
elif new in kwargs:
|
||||
if isinstance(kwargs[new], _router_class()):
|
||||
_note_renamed(label, f"a Router as {new}=", f"pass {new}=", stacklevel=4)
|
||||
kwargs[new] = _routing_context_of(kwargs[new])
|
||||
elif len(args) > index and isinstance(args[index], _router_class()):
|
||||
_note_renamed(label, "passing a Router", f"pass {new}", stacklevel=4)
|
||||
return (*args[:index], _routing_context_of(args[index]), *args[index + 1 :])
|
||||
return args
|
||||
|
||||
# An async fn keeps an async wrapper, so iscoroutinefunction (and mock.patch's AsyncMock pick) still sees it
|
||||
if inspect.iscoroutinefunction(fn):
|
||||
|
||||
@functools.wraps(fn)
|
||||
async def acall(*args: object, **kwargs: object) -> object: # kwargs-ok: forwarded to fn as given
|
||||
return await fn(*convert(args, kwargs), **kwargs) # pyright: ignore[reportAny] # fn's own result
|
||||
|
||||
return acall # pyright: ignore[reportReturnType] # same call shape as fn, see functools.wraps
|
||||
|
||||
@functools.wraps(fn)
|
||||
def call(*args: object, **kwargs: object) -> object: # kwargs-ok: forwarded to fn as given
|
||||
return fn(*convert(args, kwargs), **kwargs)
|
||||
|
||||
return call # pyright: ignore[reportReturnType] # same call shape as fn, see functools.wraps
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
class RenamedRouterAttribute:
|
||||
"""
|
||||
A class attribute that keeps an old router attribute (``self.router``, ``self.litellm_router_instance``)
|
||||
readable after it became the RoutingContext attribute ``new``: reading it returns that
|
||||
RoutingContext, writing a Router stores the Router's RoutingContext, each with a DeprecationWarning.
|
||||
"""
|
||||
|
||||
def __init__(self, new: str = "ctx") -> None:
|
||||
self.new: Final = new
|
||||
self.label = ""
|
||||
|
||||
def __set_name__(self, owner: type, name: str) -> None:
|
||||
self.label = f"{owner.__module__}.{owner.__qualname__}.{name}"
|
||||
|
||||
@overload
|
||||
def __get__(self, obj: None, owner: type | None = None) -> "RenamedRouterAttribute": ...
|
||||
@overload
|
||||
def __get__(self, obj: object, owner: type | None = None) -> object: ...
|
||||
def __get__(self, obj: object | None, owner: type | None = None) -> object:
|
||||
if obj is None:
|
||||
return self
|
||||
_note_renamed(self.label, "reading this attribute", f"read .{self.new}")
|
||||
return getattr(obj, self.new) # pyright: ignore[reportAny] # the attribute named ``new`` holds the RoutingContext
|
||||
|
||||
def __set__(self, obj: object, value: object) -> None:
|
||||
_note_renamed(self.label, "setting this attribute", f"set .{self.new}")
|
||||
setattr(obj, self.new, _routing_context_of(value))
|
||||
|
||||
|
||||
def _name_members_after_router() -> None:
|
||||
# The old members were Router methods, so errors that name the method (a wrong-arity TypeError,
|
||||
# or "litellm.router.Router.<name>() argument after ** must be a mapping") keep naming it the
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
forwarded_internal_call_metadata,
|
||||
parent_session_kwargs,
|
||||
)
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
|
|
@ -29,11 +30,11 @@ if TYPE_CHECKING:
|
|||
from semantic_router.routers import SemanticRouter
|
||||
from semantic_router.routers.base import Route
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
|
||||
from litellm.types.router import PreRoutingHookResponse
|
||||
else:
|
||||
Router = Any
|
||||
RoutingContext = Any
|
||||
PreRoutingHookResponse = Any
|
||||
Route = Any
|
||||
SemanticRouter = Any
|
||||
|
|
@ -55,13 +56,15 @@ class _SemanticMatchOutcome(NamedTuple):
|
|||
|
||||
class AutoRouter(CustomLogger):
|
||||
DEFAULT_AUTO_SYNC_VALUE = "local"
|
||||
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
default_model: str,
|
||||
embedding_model: str,
|
||||
litellm_router_instance: "Router",
|
||||
ctx: "RoutingContext",
|
||||
auto_router_config_path: str | None = None,
|
||||
auto_router_config: str | None = None,
|
||||
max_input_chars: int = DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
|
||||
|
|
@ -75,7 +78,7 @@ class AutoRouter(CustomLogger):
|
|||
auto_router_config: The config to use for the auto-router. You can either use this or auto_router_config_path, not both.
|
||||
default_model: The default model to use if no route is found.
|
||||
embedding_model: The embedding model to use for the auto-router.
|
||||
litellm_router_instance: The instance of the LiteLLM Router.
|
||||
ctx: The routing context of the owning router.
|
||||
max_input_chars: Longest prompt, in characters, handed to the embedding model. Longer
|
||||
prompts are cut to this length so an embedding context window far smaller than the
|
||||
routed model's never fails the request.
|
||||
|
|
@ -95,9 +98,9 @@ class AutoRouter(CustomLogger):
|
|||
self.default_model = default_model
|
||||
self.embedding_model: str = embedding_model
|
||||
self.max_input_chars: int = max_input_chars
|
||||
self.litellm_router_instance: Router = litellm_router_instance
|
||||
self.ctx: Final = ctx
|
||||
self.encoder: LiteLLMRouterEncoder = LiteLLMRouterEncoder(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
model_name=embedding_model,
|
||||
max_input_chars=max_input_chars,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
|
||||
|
||||
from pydantic import ConfigDict
|
||||
from semantic_router.encoders import DenseEncoder
|
||||
|
|
@ -6,11 +6,12 @@ from semantic_router.encoders.base import AsymmetricDenseMixin
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
else:
|
||||
Router = Any
|
||||
RoutingContext = Any
|
||||
|
||||
|
||||
def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
|
||||
|
|
@ -19,21 +20,22 @@ def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]:
|
|||
:param embeds: The LiteLLM embedding response.
|
||||
:return: A list of embeddings.
|
||||
"""
|
||||
if not embeds or not isinstance(embeds, litellm.EmbeddingResponse) or not embeds.data:
|
||||
if not embeds or not embeds.data:
|
||||
raise ValueError("No embeddings found in LiteLLM embedding response.")
|
||||
return [x["embedding"] for x in embeds.data]
|
||||
|
||||
|
||||
class CustomDenseEncoder(DenseEncoder):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
litellm_router_instance: ClassVar = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
|
||||
|
||||
def __init__(self, litellm_router_instance: Optional["Router"] = None, **kwargs):
|
||||
# Extract litellm_router_instance from kwargs if passed there
|
||||
if "litellm_router_instance" in kwargs:
|
||||
litellm_router_instance = kwargs.pop("litellm_router_instance")
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def __init__(self, ctx: Optional["RoutingContext"] = None, **kwargs):
|
||||
if "ctx" in kwargs:
|
||||
ctx = kwargs.pop("ctx")
|
||||
|
||||
super().__init__(**kwargs)
|
||||
self.litellm_router_instance = litellm_router_instance
|
||||
self.ctx = ctx
|
||||
|
||||
|
||||
class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
|
||||
|
|
@ -46,17 +48,18 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
|
|||
|
||||
type: str = "internal_litellm_router"
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def __init__(
|
||||
self,
|
||||
litellm_router_instance: "Router",
|
||||
ctx: "RoutingContext",
|
||||
model_name: str,
|
||||
score_threshold: float | None = None,
|
||||
max_input_chars: int = 0,
|
||||
):
|
||||
"""Initialize the LiteLLMEncoder.
|
||||
|
||||
:param litellm_router_instance: The instance of the LiteLLM Router.
|
||||
:type litellm_router_instance: Router
|
||||
:param ctx: The routing context of the owning router.
|
||||
:type ctx: RoutingContext
|
||||
:param model_name: The name of the embedding model to use. Must use LiteLLM naming
|
||||
convention (e.g. "openai/text-embedding-3-small" or "mistral/mistral-embed").
|
||||
:type model_name: str
|
||||
|
|
@ -75,7 +78,7 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
|
|||
score_threshold=score_threshold if score_threshold is not None else 0.3,
|
||||
)
|
||||
self.model_name = model_name
|
||||
self.litellm_router_instance = litellm_router_instance
|
||||
self.ctx = ctx
|
||||
self.max_input_chars = max_input_chars
|
||||
|
||||
def _clamp(self, docs: list[str]) -> list[str]: # mutable-ok: embedding() takes `input: str | list`
|
||||
|
|
@ -109,45 +112,37 @@ class LiteLLMRouterEncoder(CustomDenseEncoder, AsymmetricDenseMixin):
|
|||
return await self.aencode_queries(docs, **kwargs)
|
||||
|
||||
def encode_queries(self, docs: list[str], **kwargs) -> list[list[float]]:
|
||||
if self.litellm_router_instance is None:
|
||||
raise ValueError("litellm_router_instance is not set")
|
||||
if self.ctx is None:
|
||||
raise ValueError("ctx is not set")
|
||||
try:
|
||||
embeds: Final = self.litellm_router_instance.embedding(
|
||||
input=self._clamp(docs), model=self.model_name, **kwargs
|
||||
)
|
||||
embeds: Final = self.ctx.embedding(input=self._clamp(docs), model=self.model_name, **kwargs)
|
||||
return litellm_to_list(embeds)
|
||||
except Exception as e:
|
||||
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
|
||||
|
||||
def encode_documents(self, docs: list[str], **kwargs) -> list[list[float]]:
|
||||
if self.litellm_router_instance is None:
|
||||
raise ValueError("litellm_router_instance is not set")
|
||||
if self.ctx is None:
|
||||
raise ValueError("ctx is not set")
|
||||
try:
|
||||
embeds: Final = self.litellm_router_instance.embedding(
|
||||
input=self._clamp(docs), model=self.model_name, **kwargs
|
||||
)
|
||||
embeds: Final = self.ctx.embedding(input=self._clamp(docs), model=self.model_name, **kwargs)
|
||||
return litellm_to_list(embeds)
|
||||
except Exception as e:
|
||||
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
|
||||
|
||||
async def aencode_queries(self, docs: list[str], **kwargs) -> list[list[float]]:
|
||||
if self.litellm_router_instance is None:
|
||||
raise ValueError("litellm_router_instance is not set")
|
||||
if self.ctx is None:
|
||||
raise ValueError("ctx is not set")
|
||||
try:
|
||||
embeds: Final = await self.litellm_router_instance.aembedding(
|
||||
input=self._clamp(docs), model=self.model_name, **kwargs
|
||||
)
|
||||
embeds: Final = await self.ctx.aembedding(input=self._clamp(docs), model=self.model_name, **kwargs)
|
||||
return litellm_to_list(embeds)
|
||||
except Exception as e:
|
||||
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
|
||||
|
||||
async def aencode_documents(self, docs: list[str], **kwargs) -> list[list[float]]:
|
||||
if self.litellm_router_instance is None:
|
||||
raise ValueError("litellm_router_instance is not set")
|
||||
if self.ctx is None:
|
||||
raise ValueError("ctx is not set")
|
||||
try:
|
||||
embeds: Final = await self.litellm_router_instance.aembedding(
|
||||
input=self._clamp(docs), model=self.model_name, **kwargs
|
||||
)
|
||||
embeds: Final = await self.ctx.aembedding(input=self._clamp(docs), model=self.model_name, **kwargs)
|
||||
return litellm_to_list(embeds)
|
||||
except Exception as e:
|
||||
raise ValueError(f"{self.type.capitalize()} API call failed. Error: {e}") from e
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
|
|||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
|
|
@ -140,12 +141,12 @@ from .stall_detector import detect_stalled_task
|
|||
if TYPE_CHECKING:
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
|
||||
from litellm.router_strategy.savings_baseline import Baseline
|
||||
from litellm.types.router import PreRoutingHookResponse
|
||||
else:
|
||||
Router = Any
|
||||
RoutingContext = Any
|
||||
PreRoutingHookResponse = Any
|
||||
SemanticRouter = Any
|
||||
|
||||
|
|
@ -1416,9 +1417,11 @@ class ComplexityRouter(CustomLogger):
|
|||
- Question complexity (multiple questions)
|
||||
"""
|
||||
|
||||
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
|
||||
|
||||
def _build_jev_client(self, config: OpenSourceClassifierConfig) -> JevClassifierClient | _JevConnectionError:
|
||||
if config.deployment_name is not None:
|
||||
return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name)
|
||||
return RouterJevClassifierClient(self.ctx, config.deployment_name)
|
||||
if config.provider in ("laya", "bespoke"):
|
||||
from litellm.llms.oss_decision import oss_connection
|
||||
|
||||
|
|
@ -1459,10 +1462,11 @@ class ComplexityRouter(CustomLogger):
|
|||
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
|
||||
)
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
litellm_router_instance: Router,
|
||||
ctx: RoutingContext,
|
||||
complexity_router_config: Mapping[str, object] | None = None,
|
||||
default_model: str | None = None,
|
||||
derive_savings_baseline: bool = True,
|
||||
|
|
@ -1473,7 +1477,7 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
Args:
|
||||
model_name: The name of the model/deployment using this router.
|
||||
litellm_router_instance: The LiteLLM Router instance.
|
||||
ctx: The routing context of the owning router.
|
||||
complexity_router_config: Optional configuration dict from proxy config.
|
||||
default_model: Optional default model to use if tier cannot be determined.
|
||||
derive_savings_baseline: False for callers whose decisions are never spend
|
||||
|
|
@ -1481,7 +1485,7 @@ class ComplexityRouter(CustomLogger):
|
|||
would leak deployment mappings the caller was not authorized for.
|
||||
"""
|
||||
self.model_name = model_name
|
||||
self.litellm_router_instance = litellm_router_instance
|
||||
self.ctx: Final = ctx
|
||||
self._derive_savings_baseline = derive_savings_baseline
|
||||
|
||||
# Parse config - always create a new instance to avoid singleton mutation
|
||||
|
|
@ -1698,7 +1702,7 @@ class ComplexityRouter(CustomLogger):
|
|||
if not self._derive_savings_baseline:
|
||||
return None
|
||||
if not self._savings_baseline_derived:
|
||||
self._savings_baseline = resolve_baseline(self.litellm_router_instance, self._hardest_tier_models())
|
||||
self._savings_baseline = resolve_baseline(self.ctx, self._hardest_tier_models())
|
||||
self._savings_baseline_derived = True
|
||||
return self._savings_baseline
|
||||
|
||||
|
|
@ -2406,7 +2410,7 @@ class ComplexityRouter(CustomLogger):
|
|||
)
|
||||
|
||||
budgeted: Final = await offload_token_count(fit_jev_request)(
|
||||
build_request, prompt, system_prompt, jev_input_targets(self.litellm_router_instance, config), _truncate
|
||||
build_request, prompt, system_prompt, jev_input_targets(self.ctx, config), _truncate
|
||||
)
|
||||
request: Final = budgeted.request
|
||||
if request is None:
|
||||
|
|
@ -2946,10 +2950,8 @@ class ComplexityRouter(CustomLogger):
|
|||
"originating_request_masked": masked_originating_request(request_kwargs),
|
||||
"body": {"model": llm_config.model, **payload},
|
||||
}
|
||||
classify: Final[Callable[..., Awaitable[ModelResponse | ResponsesAPIResponse]]] = (
|
||||
self.litellm_router_instance.aresponses
|
||||
if encrypted_task is not None
|
||||
else self.litellm_router_instance.acompletion
|
||||
classify: Final[Callable[..., Awaitable[ModelResponse | ResponsesAPIResponse]]] = ( # pyright: ignore[reportAssignmentType] # the context forwards the router endpoints untyped, non-streaming calls return these
|
||||
self.ctx.aresponses if encrypted_task is not None else self.ctx.acompletion
|
||||
)
|
||||
|
||||
classifier_timeout_s: Final[float] = llm_config.timeout_ms / 1000
|
||||
|
|
@ -3148,7 +3150,7 @@ class ComplexityRouter(CustomLogger):
|
|||
+ hashlib.sha256(json.dumps(cache_identity).encode()).hexdigest()
|
||||
)
|
||||
winner: Final = await claim_affinity_pin(
|
||||
self.litellm_router_instance.cache,
|
||||
self.ctx.health().store,
|
||||
cache_key,
|
||||
MappingProxyType({"model": model}),
|
||||
self.config.session_affinity_ttl_seconds,
|
||||
|
|
@ -3241,7 +3243,7 @@ class ComplexityRouter(CustomLogger):
|
|||
model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {}
|
||||
model_to_cost: Final[dict[str, float]] = {}
|
||||
for name in available_models:
|
||||
deployments = self.litellm_router_instance.deployments_named(name)
|
||||
deployments = self.ctx.registry().deployments(name, source="named")
|
||||
if not deployments:
|
||||
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
|
||||
model_to_cost[name] = 0.0
|
||||
|
|
@ -3521,7 +3523,7 @@ class ComplexityRouter(CustomLogger):
|
|||
):
|
||||
return None
|
||||
try:
|
||||
model_info: Final = self.litellm_router_instance.get_router_model_info(
|
||||
model_info: Final = self.ctx.registry().router_model_info(
|
||||
deployment=cast(dict, deployment), # cast-ok: router deployments are plain dicts
|
||||
received_model_name=group,
|
||||
)
|
||||
|
|
@ -3531,8 +3533,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return limit if isinstance(limit, int) else None
|
||||
|
||||
def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
|
||||
list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
|
||||
deployments: Final = list_models(model_name=group) if callable(list_models) else None
|
||||
deployments: Final = self.ctx.registry().deployments(model_name=group)
|
||||
return tuple(deployments) if isinstance(deployments, list) else ()
|
||||
|
||||
def _group_output_ceiling(self, group: str) -> int | None:
|
||||
|
|
@ -3803,7 +3804,7 @@ class ComplexityRouter(CustomLogger):
|
|||
return declared is True
|
||||
return model_verdict((deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name)
|
||||
|
||||
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
|
||||
deployments: Final = self.ctx.registry().deployments(model_name=model_name)
|
||||
if not deployments:
|
||||
return (model_verdict(model_name),)
|
||||
return tuple(deployment_verdict(deployment) for deployment in deployments)
|
||||
|
|
@ -4000,9 +4001,10 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
probe_kwargs: Final = dict(request_kwargs)
|
||||
try:
|
||||
deployments: Final = await self.litellm_router_instance.async_get_healthy_deployments(
|
||||
model=model_name,
|
||||
request_kwargs=probe_kwargs,
|
||||
deployments: Final = await self.ctx.select(
|
||||
model_name,
|
||||
probe_kwargs,
|
||||
kind="healthy",
|
||||
messages=messages,
|
||||
input=input,
|
||||
parent_otel_span=get_parent_otel_span_from_kwargs(request_kwargs),
|
||||
|
|
@ -4241,7 +4243,7 @@ class ComplexityRouter(CustomLogger):
|
|||
routelayer: Final = SemanticRouter(
|
||||
routes=routes,
|
||||
encoder=LiteLLMRouterEncoder(
|
||||
litellm_router_instance=self.litellm_router_instance,
|
||||
ctx=self.ctx,
|
||||
model_name=embedding_model,
|
||||
score_threshold=self.config.match_threshold,
|
||||
),
|
||||
|
|
@ -4535,7 +4537,7 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
if cache_key is not None and pin_replay_allowed:
|
||||
pinned_value: Final = await read_affinity_pin(
|
||||
self.litellm_router_instance.cache, cache_key, self.config.session_affinity_ttl_seconds
|
||||
self.ctx.health().store, cache_key, self.config.session_affinity_ttl_seconds
|
||||
)
|
||||
pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names())
|
||||
if pinned_pin is not None:
|
||||
|
|
@ -4634,7 +4636,7 @@ class ComplexityRouter(CustomLogger):
|
|||
else floor_model
|
||||
)
|
||||
if escalated:
|
||||
await self.litellm_router_instance.cache.async_set_cache(
|
||||
await self.ctx.health().store.async_set_cache(
|
||||
key=cache_key,
|
||||
value=_session_affinity_cache_value(session_model, resolved_pin_tier),
|
||||
ttl=self.config.session_affinity_ttl_seconds,
|
||||
|
|
@ -4737,7 +4739,7 @@ class ComplexityRouter(CustomLogger):
|
|||
and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None
|
||||
)
|
||||
if pinnable and cache_key is not None and response is not None:
|
||||
await self.litellm_router_instance.cache.async_set_cache(
|
||||
await self.ctx.health().store.async_set_cache(
|
||||
key=cache_key,
|
||||
value=_session_affinity_cache_value(
|
||||
response.model,
|
||||
|
|
@ -4759,7 +4761,7 @@ class ComplexityRouter(CustomLogger):
|
|||
from litellm.proxy.common_utils.cache_aware_routing import choose_cached_model
|
||||
|
||||
choice: Final = await choose_cached_model(
|
||||
router=self.litellm_router_instance,
|
||||
ctx=self.ctx,
|
||||
config=self.config,
|
||||
params_for_model=partial(self._litellm_params_for_model, request_kwargs=request_kwargs),
|
||||
response=response,
|
||||
|
|
|
|||
|
|
@ -27,10 +27,11 @@ from litellm.llms.compaction import (
|
|||
dispatch,
|
||||
get_native_compaction_provider,
|
||||
)
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_strategy.complexity_router.config import ContextCompactionConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
Surface: TypeAlias = Literal["chat", "messages", "responses"]
|
||||
_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType(
|
||||
|
|
@ -96,13 +97,14 @@ def initialize_compaction_state(kwargs: Mapping[str, object], surface: Surface)
|
|||
return existing if isinstance(existing, CompactionState) else CompactionState(surface=surface)
|
||||
|
||||
|
||||
@renamed_router_param("router")
|
||||
async def arm_compaction(
|
||||
kwargs: Mapping[str, object],
|
||||
config: ContextCompactionConfig | Literal[False] | None,
|
||||
candidates: tuple[str, ...] = (),
|
||||
parent_model: str | None = None,
|
||||
*,
|
||||
router: Router | None = None,
|
||||
ctx: RoutingContext | None = None,
|
||||
allow_escalation: bool = False,
|
||||
messages: Sequence[Mapping[str, object]] | None = None,
|
||||
) -> None:
|
||||
|
|
@ -111,7 +113,7 @@ async def arm_compaction(
|
|||
state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None
|
||||
state.candidates = candidates
|
||||
state.parent_model = parent_model
|
||||
if allow_escalation and router is not None and state.config is not None:
|
||||
if allow_escalation and ctx is not None and state.config is not None:
|
||||
payload: Final = MappingProxyType(
|
||||
{
|
||||
**kwargs,
|
||||
|
|
@ -119,11 +121,11 @@ async def arm_compaction(
|
|||
**({"messages": messages} if messages is not None and state.surface != "responses" else {}),
|
||||
}
|
||||
)
|
||||
if not await _has_compactor(router, state, payload):
|
||||
if not await _has_compactor(ctx, state, payload):
|
||||
state.config = None
|
||||
|
||||
|
||||
async def _has_compactor(router: Router, state: CompactionState, payload: Mapping[str, object]) -> bool:
|
||||
async def _has_compactor(ctx: RoutingContext, state: CompactionState, payload: Mapping[str, object]) -> bool:
|
||||
from litellm.exceptions import ContextWindowExceededError
|
||||
|
||||
if state.surface is None or state.config is None:
|
||||
|
|
@ -131,7 +133,7 @@ async def _has_compactor(router: Router, state: CompactionState, payload: Mappin
|
|||
try:
|
||||
instructions, prefix, _ = _portable_history(payload, state.surface)
|
||||
await _compactor_model(
|
||||
router, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens)
|
||||
ctx, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens)
|
||||
)
|
||||
return True
|
||||
except ContextWindowExceededError:
|
||||
|
|
@ -237,9 +239,9 @@ def _history(
|
|||
return instructions, prefix, conversation[split:]
|
||||
|
||||
|
||||
async def _count(router: Router, payload: Mapping[str, object]) -> int:
|
||||
async def _count(ctx: RoutingContext, payload: Mapping[str, object]) -> int:
|
||||
return await asyncio.to_thread(
|
||||
router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter
|
||||
ctx.count_pre_call_check_tokens,
|
||||
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None,
|
||||
input=_INPUT.validate_python(payload.get("input")),
|
||||
request_kwargs=payload,
|
||||
|
|
@ -247,11 +249,11 @@ async def _count(router: Router, payload: Mapping[str, object]) -> int:
|
|||
|
||||
|
||||
def _budget(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
|
||||
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float
|
||||
) -> InputBudget:
|
||||
model: Final = str(payload.get("model", ""))
|
||||
info: Final = _MAPPING.validate_python(
|
||||
router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
|
||||
ctx.registry().router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model)
|
||||
)
|
||||
raw_window: Final = info.get("max_input_tokens")
|
||||
window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None
|
||||
|
|
@ -269,9 +271,9 @@ def _budget(
|
|||
|
||||
|
||||
async def _compactor_model(
|
||||
router: Router, state: CompactionState, payload: Mapping[str, object]
|
||||
ctx: RoutingContext, state: CompactionState, payload: Mapping[str, object]
|
||||
) -> tuple[str, NativeCompactionProvider]:
|
||||
needed: Final = await _count(router, payload)
|
||||
needed: Final = await _count(ctx, payload)
|
||||
candidates: Final = (
|
||||
(state.config.model,) if state.config is not None and state.config.model is not None else state.candidates
|
||||
)
|
||||
|
|
@ -279,13 +281,13 @@ async def _compactor_model(
|
|||
(
|
||||
(candidate, provider)
|
||||
for candidate in candidates
|
||||
if (deployments := tuple(router.get_model_list(model_name=candidate) or ()))
|
||||
if (deployments := tuple(ctx.registry().deployments(model_name=candidate) or ()))
|
||||
and (provider := get_native_compaction_provider(_MAPPING.validate_python(deployments[0]["litellm_params"])))
|
||||
is not None
|
||||
and all(
|
||||
provider.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"]))
|
||||
and provider.compatible_defaults(params)
|
||||
and (budget := _budget(router, deployment, payload, 0.9)).available is not None
|
||||
and (budget := _budget(ctx, deployment, payload, 0.9)).available is not None
|
||||
and needed <= budget.available
|
||||
for deployment in deployments
|
||||
)
|
||||
|
|
@ -320,7 +322,7 @@ def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[s
|
|||
|
||||
|
||||
async def _generate_summary(
|
||||
router: Router,
|
||||
ctx: RoutingContext,
|
||||
provider: NativeCompactionProvider,
|
||||
protocol: CompactionProtocol,
|
||||
payload: Mapping[str, object],
|
||||
|
|
@ -330,7 +332,7 @@ async def _generate_summary(
|
|||
executor: Final = compaction_executor.get()
|
||||
with native_compaction_call():
|
||||
response: Final = await asyncio.wait_for(
|
||||
executor(protocol, payload, parent_model) if executor is not None else dispatch(router, protocol, payload),
|
||||
executor(protocol, payload, parent_model) if executor is not None else dispatch(ctx, protocol, payload),
|
||||
timeout=timeout,
|
||||
)
|
||||
summary: Final = provider.extract_summary(protocol, response)
|
||||
|
|
@ -362,32 +364,33 @@ def _compactor_input(
|
|||
)
|
||||
|
||||
|
||||
@renamed_router_param("router")
|
||||
async def compact_to_fit(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
|
||||
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
|
||||
) -> Mapping[str, object]:
|
||||
from litellm.exceptions import ContextWindowExceededError
|
||||
|
||||
try:
|
||||
return await _compact_to_fit(router, deployment, payload, surface)
|
||||
return await _compact_to_fit(ctx, deployment, payload, surface)
|
||||
except ContextWindowExceededError:
|
||||
window: Final = _budget(router, deployment, payload, 1.0).window
|
||||
if not _native_child.get() and window is not None and await _count(router, payload) <= window:
|
||||
window: Final = _budget(ctx, deployment, payload, 1.0).window
|
||||
if not _native_child.get() and window is not None and await _count(ctx, payload) <= window:
|
||||
return payload
|
||||
raise
|
||||
|
||||
|
||||
async def _check_client_managed_admission(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object]
|
||||
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object]
|
||||
) -> None:
|
||||
if not router.enable_pre_call_checks:
|
||||
if not ctx.settings().enable_pre_call_checks:
|
||||
return
|
||||
router._pre_call_checks( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # restore legacy admission after deployment defaults
|
||||
ctx.pre_call_checks(
|
||||
model=str(payload["model"]),
|
||||
healthy_deployments=_ITEMS.validate_python((deployment,)),
|
||||
healthy_deployments=list[object](_ITEMS.validate_python((deployment,))),
|
||||
messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, # pyright: ignore[reportArgumentType] # legacy annotation omits structured content
|
||||
input=_INPUT.validate_python(payload.get("input")),
|
||||
request_kwargs=_DICT.validate_python(payload),
|
||||
input_token_count=await _count(router, payload),
|
||||
input_token_count=await _count(ctx, payload),
|
||||
skip_inline_token_count=True,
|
||||
)
|
||||
|
||||
|
|
@ -419,22 +422,22 @@ def _portable_history(
|
|||
|
||||
|
||||
async def _compact_to_fit(
|
||||
router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
|
||||
ctx: RoutingContext, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None
|
||||
) -> Mapping[str, object]:
|
||||
state: Final = payload.get(_STATE_KEY)
|
||||
config: Final = state.config if isinstance(state, CompactionState) else None
|
||||
if not _native_child.get() and (config is None or _client_managed(payload)):
|
||||
if config is not None:
|
||||
await _check_client_managed_admission(router, deployment, payload)
|
||||
await _check_client_managed_admission(ctx, deployment, payload)
|
||||
return payload
|
||||
model: Final = str(payload["model"])
|
||||
limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9)
|
||||
limits: Final = _budget(ctx, deployment, payload, config.trigger_ratio if config is not None else 0.9)
|
||||
budget: Final = limits.available
|
||||
if budget is None or budget <= 0:
|
||||
if (
|
||||
config is not None
|
||||
and config.model is None
|
||||
and (limits.window is None or await _count(router, payload) <= limits.window)
|
||||
and (limits.window is None or await _count(ctx, payload) <= limits.window)
|
||||
):
|
||||
return payload
|
||||
_unavailable(model, "A known input window and a smaller output allowance are required")
|
||||
|
|
@ -443,18 +446,18 @@ async def _compact_to_fit(
|
|||
if (
|
||||
child_provider is None
|
||||
or not child_provider.compatible_defaults(payload)
|
||||
or await _count(router, payload) > budget
|
||||
or await _count(ctx, payload) > budget
|
||||
):
|
||||
_reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget")
|
||||
return payload
|
||||
if await _count(router, payload) <= budget:
|
||||
if await _count(ctx, payload) <= budget:
|
||||
return payload
|
||||
if surface is None or config is None or not isinstance(state, CompactionState):
|
||||
_unavailable(model, "This request surface cannot be compacted")
|
||||
key: Final = "input" if surface == "responses" else "messages"
|
||||
instructions, prefix, tail = _portable_history(payload, surface)
|
||||
retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))})
|
||||
if await _count(router, retained) >= budget:
|
||||
if await _count(ctx, retained) >= budget:
|
||||
_unavailable(model, "Retained instructions, tools and the latest turn leave no room for a summary")
|
||||
older: Final = _compactor_input(payload, surface, instructions, prefix, config.max_tokens)
|
||||
metadata: Final = sanitized_forwardable_call_metadata(
|
||||
|
|
@ -481,7 +484,7 @@ async def _compact_to_fit(
|
|||
**parent_session_kwargs(payload),
|
||||
}
|
||||
)
|
||||
compactor, provider = await _compactor_model(router, state, request)
|
||||
compactor, provider = await _compactor_model(ctx, state, request)
|
||||
child: Final = MappingProxyType({**request, **provider.request_kwargs(), "model": compactor})
|
||||
identity: Final = hashlib.sha256(
|
||||
json.dumps(
|
||||
|
|
@ -505,7 +508,7 @@ async def _compact_to_fit(
|
|||
state.summary = (
|
||||
identity,
|
||||
asyncio.create_task(
|
||||
_generate_summary(router, provider, protocol, child, config.timeout_seconds, state.parent_model)
|
||||
_generate_summary(ctx, provider, protocol, child, config.timeout_seconds, state.parent_model)
|
||||
),
|
||||
)
|
||||
if state.summary[0] != identity:
|
||||
|
|
@ -515,6 +518,6 @@ async def _compact_to_fit(
|
|||
{"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary}
|
||||
)
|
||||
compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))})
|
||||
if await _count(router, compacted) > budget:
|
||||
if await _count(ctx, compacted) > budget:
|
||||
_unavailable(model, "The summary and retained conversation still exceed the selected deployment's budget")
|
||||
return compacted
|
||||
|
|
|
|||
|
|
@ -239,10 +239,10 @@ def run_eval() -> tuple[int, int, list[dict]]:
|
|||
Tuple of (passed, total, failures)
|
||||
"""
|
||||
# Create router with default config
|
||||
mock_router: Final = MagicMock()
|
||||
mock_ctx: Final = MagicMock()
|
||||
router: Final = ComplexityRouter(
|
||||
model_name="eval-router",
|
||||
litellm_router_instance=mock_router,
|
||||
ctx=mock_ctx,
|
||||
)
|
||||
|
||||
passed = 0
|
||||
|
|
|
|||
|
|
@ -23,13 +23,13 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
|||
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG, is_ai_decide_model
|
||||
from litellm.llms.laya.common_utils import laya_response_model
|
||||
from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS
|
||||
from litellm.types.decisions import DecisionsResponse
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, StandardLoggingRoutingDecision
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_strategy.complexity_router.config import OpenSourceClassifierConfig
|
||||
from litellm.types.router import Deployment
|
||||
|
|
@ -68,14 +68,12 @@ class JevBudgetedRequest:
|
|||
telemetry: StandardLoggingRoutingDecision
|
||||
|
||||
|
||||
def jev_input_targets(router: "Router", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
|
||||
from litellm.router_api import RoutingContext
|
||||
def jev_input_targets(context: "RoutingContext", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]:
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
if config.deployment_name is None:
|
||||
provider: Final = "typesafe" if config.provider == "jev" else config.provider
|
||||
return (_jev_input_target({"model": f"{provider}/{config.model}"}, None),)
|
||||
context: Final = RoutingContext(router)
|
||||
deployments: Final = context.registry().deployments(model_name=config.deployment_name) or ()
|
||||
return tuple(
|
||||
_jev_deployment_input_target(context, Deployment.model_validate(deployment)) for deployment in deployments
|
||||
|
|
@ -240,30 +238,13 @@ class JevClassifierClient(Protocol):
|
|||
) -> JevSystemOneResponse: ...
|
||||
|
||||
|
||||
class _DecisionsRouter(Protocol):
|
||||
async def adecisions(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
state: str,
|
||||
questions: Mapping[str, object],
|
||||
timeout: float,
|
||||
num_retries: int,
|
||||
disable_fallbacks: bool,
|
||||
metadata: Mapping[str, object],
|
||||
proxy_server_request: Mapping[str, object],
|
||||
turn_off_message_logging: bool | None,
|
||||
litellm_session_id: str = "",
|
||||
litellm_trace_id: str = "",
|
||||
) -> object: ...
|
||||
|
||||
|
||||
_PARENT_METADATA: Final = TypeAdapter[Mapping[str, object] | None](Mapping[str, object] | None)
|
||||
|
||||
|
||||
class RouterJevClassifierClient:
|
||||
def __init__(self, router: "Router", deployment_name: str) -> None:
|
||||
self._router: _DecisionsRouter = router
|
||||
@renamed_router_param("router")
|
||||
def __init__(self, ctx: "RoutingContext", deployment_name: str) -> None:
|
||||
self.ctx: Final = ctx
|
||||
self._deployment_name = deployment_name
|
||||
|
||||
async def evaluate(
|
||||
|
|
@ -285,7 +266,7 @@ class RouterJevClassifierClient:
|
|||
"questions": {key: value.model_dump() for key, value in request.questions.items()},
|
||||
}
|
||||
response: Final = DecisionsResponse.model_validate(
|
||||
await self._router.adecisions(
|
||||
await self.ctx.adecisions(
|
||||
model=self._deployment_name,
|
||||
state=request.state,
|
||||
questions={key: value.model_dump() for key, value in request.questions.items()},
|
||||
|
|
|
|||
|
|
@ -16,10 +16,12 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router import CustomRoutingStrategyBase
|
||||
from litellm.router_api import ChangeSet
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.types.lar1 import LAR1Metadata, LAR1Time
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
DEFAULT_THRESHOLDS: Final[dict[str, float]] = {"low": 0.3, "medium": 0.5, "high": 0.7}
|
||||
|
||||
|
|
@ -41,16 +43,17 @@ def lar1_thresholds_from_args(
|
|||
}
|
||||
|
||||
|
||||
@renamed_router_param("router")
|
||||
def apply_lar1_routing_strategy(
|
||||
router: Router,
|
||||
ctx: RoutingContext,
|
||||
routing_strategy_args: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
strategy: Final = LAR1RoutingStrategy(
|
||||
router_instance=router,
|
||||
ctx=ctx,
|
||||
thresholds=lar1_thresholds_from_args(routing_strategy_args),
|
||||
)
|
||||
router.routing_strategy = "lar1"
|
||||
router.set_custom_routing_strategy(strategy)
|
||||
ctx.apply(ChangeSet(assign={"routing_strategy": "lar1"}))
|
||||
ctx.register_plugin(strategy)
|
||||
|
||||
|
||||
def _normalize_thresholds(thresholds: dict[str, float] | None) -> dict[str, float]:
|
||||
|
|
@ -78,12 +81,13 @@ def _parse_lar1_metadata(request_kwargs: dict) -> LAR1Metadata:
|
|||
|
||||
|
||||
class LAR1RoutingStrategy(CustomRoutingStrategyBase):
|
||||
@renamed_router_param("router_instance")
|
||||
def __init__(
|
||||
self,
|
||||
router_instance: Router | None = None,
|
||||
ctx: RoutingContext | None = None,
|
||||
thresholds: dict[str, float] | None = None,
|
||||
):
|
||||
self._router = router_instance
|
||||
self._ctx = ctx
|
||||
self.thresholds = _normalize_thresholds(thresholds)
|
||||
|
||||
async def async_get_available_deployment(
|
||||
|
|
@ -96,7 +100,7 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
|
|||
):
|
||||
if request_kwargs is None:
|
||||
request_kwargs = {}
|
||||
if self._router is None:
|
||||
if self._ctx is None:
|
||||
return None
|
||||
|
||||
lar1: Final = _parse_lar1_metadata(request_kwargs)
|
||||
|
|
@ -104,9 +108,10 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
|
|||
evidence: Final = tuple(e.value for e in lar1.evidence)
|
||||
time_dim: Final = lar1.time.value
|
||||
|
||||
healthy: Final = await self._router.async_get_healthy_deployments(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
healthy: Final = await self._ctx.select(
|
||||
model,
|
||||
request_kwargs,
|
||||
kind="healthy",
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
|
|
@ -114,7 +119,7 @@ class LAR1RoutingStrategy(CustomRoutingStrategyBase):
|
|||
if isinstance(healthy, dict):
|
||||
return healthy
|
||||
|
||||
if not healthy:
|
||||
if not isinstance(healthy, list) or not healthy:
|
||||
return None
|
||||
|
||||
target: Final = self._classify_request(confidence, evidence, time_dim)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
|
||||
|
||||
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional
|
|||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
from litellm.router_strategy.complexity_router.complexity_router import (
|
||||
ComplexityRouter,
|
||||
)
|
||||
|
|
@ -29,10 +30,10 @@ from litellm.types.utils import StandardLoggingRoutingDecision
|
|||
from .config import QualityRouterConfig, RoutingPreferences
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.types.router import PreRoutingHookResponse
|
||||
else:
|
||||
Router = Any
|
||||
RoutingContext = Any
|
||||
PreRoutingHookResponse = Any
|
||||
|
||||
|
||||
|
|
@ -42,15 +43,18 @@ class QualityRouter(CustomLogger):
|
|||
keyword override.
|
||||
"""
|
||||
|
||||
litellm_router_instance: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
litellm_router_instance: "Router",
|
||||
ctx: "RoutingContext",
|
||||
default_model: str | None = None,
|
||||
quality_router_config: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.litellm_router_instance = litellm_router_instance
|
||||
self.ctx: Final = ctx
|
||||
|
||||
if quality_router_config:
|
||||
self.config = QualityRouterConfig.model_validate(dict(**quality_router_config))
|
||||
|
|
@ -64,7 +68,7 @@ class QualityRouter(CustomLogger):
|
|||
# Internal scorer — re-use the existing rule-based classifier.
|
||||
self._scorer = ComplexityRouter(
|
||||
model_name=f"{model_name}::scorer",
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
)
|
||||
|
||||
# Per-model indices populated alongside the tier index. `_model_keywords`
|
||||
|
|
@ -158,7 +162,7 @@ class QualityRouter(CustomLogger):
|
|||
`_model_quality`, and `_model_cost`. Raises if any listed model is
|
||||
missing `litellm_routing_preferences`.
|
||||
"""
|
||||
model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or []
|
||||
model_list: Final = self.ctx.registry().all() or []
|
||||
available: Final = set(self.config.available_models)
|
||||
|
||||
# Track which available models we've matched so we can error on missing.
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ from collections.abc import Iterable, Mapping
|
|||
from typing import TYPE_CHECKING, Final, NamedTuple
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
|
||||
_REFERENCE_REQUEST: Final = Usage(
|
||||
|
|
@ -70,7 +71,7 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
|
|||
return f"{provider}/{resolved}"
|
||||
|
||||
|
||||
def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
|
||||
def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
|
||||
"""The candidates a tier entry actually calls, each with its own pricing key.
|
||||
|
||||
`litellm_params.model` is not always a model: on Azure it is the deployment name,
|
||||
|
|
@ -80,7 +81,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
|
|||
straight at a provider model rather than at a configured group, and prices under
|
||||
its own name because there is no deployment to override it.
|
||||
"""
|
||||
deployments: Final = router.deployments_named(group_name)
|
||||
deployments: Final = ctx.registry().deployments(group_name, source="named")
|
||||
if not deployments:
|
||||
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
|
||||
|
||||
|
|
@ -100,7 +101,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
|
|||
return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None)
|
||||
|
||||
|
||||
def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None:
|
||||
def _priced(ctx: "RoutingContext", candidate: Baseline) -> tuple[float, Baseline] | None:
|
||||
"""``(cost_of_the_reference_request, candidate)``, or ``None`` when unpriceable.
|
||||
|
||||
"Most expensive" is a property of a request, not of a rate: a deployment dearer per
|
||||
|
|
@ -114,7 +115,7 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
|
|||
|
||||
provider, _, model_name = candidate.model.partition("/")
|
||||
try:
|
||||
info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model)
|
||||
info: Final = ctx.registry().deployment_model_info(candidate.deployment_id or "", candidate.model, cached=True)
|
||||
if info is None:
|
||||
return None
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
|
|
@ -133,16 +134,17 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
|
|||
return (cost, candidate)
|
||||
|
||||
|
||||
def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None:
|
||||
def _most_expensive(ctx: "RoutingContext", candidates: Iterable[Baseline]) -> Baseline | None:
|
||||
"""The candidate that would have cost the most on the reference request."""
|
||||
priced: Final = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None)
|
||||
priced: Final = tuple(r for candidate in candidates if (r := _priced(ctx, candidate)) is not None)
|
||||
if not priced:
|
||||
verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled")
|
||||
return None
|
||||
return max(priced)[1]
|
||||
|
||||
|
||||
def resolve_baseline(router: "Router", group_names: Iterable[str]) -> Baseline | None:
|
||||
@renamed_router_param("router")
|
||||
def resolve_baseline(ctx: "RoutingContext", group_names: Iterable[str]) -> Baseline | None:
|
||||
"""The derived baseline for a router whose hardest tier offers ``group_names``.
|
||||
|
||||
Holds no cache of its own; each pricing pass walks the pool, so the caller is
|
||||
|
|
@ -155,7 +157,7 @@ def resolve_baseline(router: "Router", group_names: Iterable[str]) -> Baseline |
|
|||
over; an unresolvable baseline zeroes the savings driver instead.
|
||||
"""
|
||||
try:
|
||||
return _most_expensive(router, (c for name in group_names for c in _models_in(router, name)))
|
||||
return _most_expensive(ctx, (c for name in group_names for c in _models_in(ctx, name)))
|
||||
except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric
|
||||
verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import re
|
|||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypedDict, overload
|
||||
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
|
|
@ -23,14 +23,11 @@ from litellm.constants import (
|
|||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.types.router import BaggageMappedTagsStamp, ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
|
||||
class _TagLitellmParamsLike(Protocol):
|
||||
|
|
@ -394,18 +391,18 @@ def _resolve_constraint_only_pool(
|
|||
|
||||
|
||||
def _all_deployments_or_fallback(
|
||||
llm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model: str,
|
||||
fallback: _DeploymentPool,
|
||||
) -> Sequence[_DeploymentLike | DeploymentTypedDict] | Mapping[_DeploymentLike, object]:
|
||||
try:
|
||||
return llm_router_instance.get_all_deployments(model_name=model)
|
||||
return ctx.registry().deployments(model, source="all")
|
||||
except Exception: # noqa: BLE001 # fail safe toward today's healthy-only behavior on lookup errors
|
||||
return fallback
|
||||
|
||||
|
||||
def _chain_tag_filtering_override(
|
||||
llm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model: str,
|
||||
healthy_deployments: _DeploymentPool,
|
||||
) -> bool | None:
|
||||
|
|
@ -418,7 +415,7 @@ def _chain_tag_filtering_override(
|
|||
# repeatedly failing that one deployment into cooldown. Falls back to
|
||||
# healthy_deployments on a lookup error, preserving today's behavior rather
|
||||
# than crashing the request.
|
||||
all_deployments: Final = _all_deployments_or_fallback(llm_router_instance, model, healthy_deployments)
|
||||
all_deployments: Final = _all_deployments_or_fallback(ctx, model, healthy_deployments)
|
||||
for d in all_deployments:
|
||||
value = (d.get("model_info") or MappingProxyType({})).get("enable_tag_filtering")
|
||||
if value is not None:
|
||||
|
|
@ -458,7 +455,7 @@ def _tags_that_route(request_tags: Sequence[str], metadata: object, routing_pref
|
|||
|
||||
|
||||
def _tag_known_to_group(
|
||||
llm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model: str,
|
||||
positive_tags: Sequence[str],
|
||||
routing_confirmed: frozenset[str],
|
||||
|
|
@ -467,7 +464,7 @@ def _tag_known_to_group(
|
|||
if tag_set & routing_confirmed:
|
||||
return True
|
||||
try:
|
||||
all_deployments: Final = llm_router_instance.get_all_deployments(model_name=model)
|
||||
all_deployments: Final = ctx.registry().deployments(model, source="all")
|
||||
except Exception: # noqa: BLE001 # fail safe toward "unrecognized" so lookup errors preserve the existing silent-fallback behavior
|
||||
return False
|
||||
return any(
|
||||
|
|
@ -557,8 +554,9 @@ def _tag_sequence(tags: object) -> Sequence[object]:
|
|||
return typed_tags
|
||||
|
||||
|
||||
@renamed_router_param("llm_router_instance")
|
||||
async def get_deployments_for_tag(
|
||||
llm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model: str, # used to raise the correct error
|
||||
healthy_deployments: _DeploymentPool,
|
||||
request_kwargs: _RequestKwargsLike | None = None,
|
||||
|
|
@ -587,11 +585,9 @@ async def get_deployments_for_tag(
|
|||
return healthy_deployments
|
||||
|
||||
request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering")
|
||||
chain_enable_tag_filtering: Final = _chain_tag_filtering_override(llm_router_instance, model, healthy_deployments)
|
||||
chain_enable_tag_filtering: Final = _chain_tag_filtering_override(ctx, model, healthy_deployments)
|
||||
chain_default: Final = (
|
||||
chain_enable_tag_filtering
|
||||
if chain_enable_tag_filtering is not None
|
||||
else llm_router_instance.enable_tag_filtering
|
||||
chain_enable_tag_filtering if chain_enable_tag_filtering is not None else ctx.settings().enable_tag_filtering
|
||||
)
|
||||
if request_enable_tag_filtering is not True and chain_default is not True:
|
||||
return healthy_deployments
|
||||
|
|
@ -602,8 +598,8 @@ async def get_deployments_for_tag(
|
|||
request_tags: Final = without_baggage_mapped_tags(
|
||||
_request_tags_after_router_consumption(metadata, model), metadata
|
||||
)
|
||||
match_any: Final = llm_router_instance.tag_filtering_match_any
|
||||
routing_prefix: Final = llm_router_instance.tag_routing_prefix or ""
|
||||
match_any: Final = ctx.settings().tag_filtering_match_any
|
||||
routing_prefix: Final = ctx.settings().tag_routing_prefix or ""
|
||||
|
||||
# Build header strings for regex matching from what the proxy already stores.
|
||||
# Currently we match against User-Agent; format matches "^User-Agent: claude-code/..."
|
||||
|
|
@ -703,7 +699,7 @@ async def get_deployments_for_tag(
|
|||
if (
|
||||
len(new_healthy_deployments) == 0
|
||||
and positive_tags
|
||||
and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed)
|
||||
and _tag_known_to_group(ctx, model, positive_tags, routing_confirmed)
|
||||
):
|
||||
return _resolve_or_fail_open(
|
||||
request_deployments,
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.utils import calculate_max_parallel_requests
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
|
||||
class MaxParallelRequestsLimit:
|
||||
|
|
@ -52,7 +49,8 @@ class MaxParallelRequestsLimit:
|
|||
|
||||
class InitalizeCachedClient:
|
||||
@staticmethod
|
||||
def set_max_parallel_requests_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def set_max_parallel_requests_client(ctx: "RoutingContext", model: dict):
|
||||
litellm_params: Final = model.get("litellm_params", {})
|
||||
model_id: Final = model["model_info"]["id"]
|
||||
rpm: Final = litellm_params.get("rpm", None)
|
||||
|
|
@ -62,7 +60,7 @@ class InitalizeCachedClient:
|
|||
rpm=rpm,
|
||||
max_parallel_requests=max_parallel_requests,
|
||||
tpm=tpm,
|
||||
default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests,
|
||||
default_max_parallel_requests=ctx.settings().default_max_parallel_requests,
|
||||
)
|
||||
if calculated_max_parallel_requests:
|
||||
limit: Final = MaxParallelRequestsLimit(
|
||||
|
|
@ -71,7 +69,7 @@ class InitalizeCachedClient:
|
|||
model_group=model.get("model_name", ""),
|
||||
)
|
||||
cache_key: Final = f"{model_id}_max_parallel_requests_client"
|
||||
litellm_router_instance.cache.set_cache(
|
||||
ctx.health().store.set_cache(
|
||||
key=cache_key,
|
||||
value=limit,
|
||||
local_only=True,
|
||||
|
|
|
|||
|
|
@ -7,19 +7,18 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.router_api import RoutingContext
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
PrometheusLogger = Any
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def router_cooldown_event_callback(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment_id: str,
|
||||
exception_status: str | int,
|
||||
cooldown_time: float | None,
|
||||
|
|
@ -31,7 +30,7 @@ async def router_cooldown_event_callback(
|
|||
- Increments cooldown metric for deployment on Prometheus
|
||||
"""
|
||||
verbose_logger.debug("In router_cooldown_event_callback - updating prometheus")
|
||||
_deployment: Final = litellm_router_instance.get_deployment(model_id=deployment_id)
|
||||
_deployment: Final = ctx.registry().deployment(deployment_id)
|
||||
if _deployment is None:
|
||||
verbose_logger.warning(
|
||||
"in router_cooldown_event_callback but _deployment is None for deployment_id=%s. Doing nothing",
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.constants import (
|
|||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue
|
||||
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
|
||||
from litellm.types.router import Deployment
|
||||
|
|
@ -48,12 +49,10 @@ from .router_callbacks.track_deployment_metrics import (
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.router import Router as _Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
LitellmRouter = _Router
|
||||
Span = _Span
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
Span = Any
|
||||
|
||||
_ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
|
||||
|
|
@ -137,7 +136,7 @@ def _first_present(*sources: Mapping[str, Any] | None, key: str) -> int | float
|
|||
|
||||
|
||||
def _get_deployment_cooldown_policy(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str,
|
||||
) -> tuple[Mapping[str, int] | None, int | None]:
|
||||
"""Return (allowed_fails_policy, allowed_fails) from deployment model_info, or (None, None).
|
||||
|
|
@ -149,7 +148,7 @@ def _get_deployment_cooldown_policy(
|
|||
field placed there would leak into the outgoing LLM request instead of staying
|
||||
router-internal.
|
||||
"""
|
||||
dep: Final = litellm_router_instance.get_model_info(id=deployment)
|
||||
dep: Final = ctx.registry().deployment_dict(deployment)
|
||||
if dep is None:
|
||||
return None, None
|
||||
mi: Final[Mapping[str, Any]] = dep.get("model_info") or MappingProxyType({})
|
||||
|
|
@ -175,7 +174,7 @@ def _resolve_allowed_fails_from_policy(
|
|||
|
||||
|
||||
def _should_cooldown_based_on_deployment_policy(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str,
|
||||
original_exception: Exception,
|
||||
dep_policy: Mapping[str, int] | None,
|
||||
|
|
@ -199,10 +198,7 @@ def _should_cooldown_based_on_deployment_policy(
|
|||
if (
|
||||
allowed_fails_from_policy is None
|
||||
and is_single_deployment_model_group
|
||||
and (
|
||||
dep_allowed_fails is not None
|
||||
or litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None
|
||||
)
|
||||
and (dep_allowed_fails is not None or ctx.settings().allowed_fails_for(original_exception) is None)
|
||||
):
|
||||
return False
|
||||
|
||||
|
|
@ -215,7 +211,7 @@ def _should_cooldown_based_on_deployment_policy(
|
|||
else ("generic" if dep_allowed_fails is not None else None)
|
||||
)
|
||||
|
||||
dep: Final = litellm_router_instance.get_model_info(id=deployment)
|
||||
dep: Final = ctx.registry().deployment_dict(deployment)
|
||||
cooldown_time_override: Final = (
|
||||
_first_present(dep.get("model_info"), dep.get("litellm_params"), key="cooldown_time")
|
||||
if dep is not None
|
||||
|
|
@ -223,7 +219,7 @@ def _should_cooldown_based_on_deployment_policy(
|
|||
)
|
||||
|
||||
return should_cooldown_based_on_allowed_fails_policy(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment=deployment,
|
||||
original_exception=original_exception,
|
||||
allowed_fails_override=allowed_fails_override,
|
||||
|
|
@ -233,7 +229,7 @@ def _should_cooldown_based_on_deployment_policy(
|
|||
|
||||
|
||||
def _has_explicit_allowed_fails_policy_for_exception(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str | None,
|
||||
original_exception: Exception,
|
||||
) -> bool:
|
||||
|
|
@ -254,12 +250,12 @@ def _has_explicit_allowed_fails_policy_for_exception(
|
|||
"""
|
||||
if deployment is None:
|
||||
return False
|
||||
dep_policy, _ = _get_deployment_cooldown_policy(litellm_router_instance, deployment)
|
||||
dep_policy, _ = _get_deployment_cooldown_policy(ctx, deployment)
|
||||
return _resolve_allowed_fails_from_policy(dep_policy, original_exception) is not None
|
||||
|
||||
|
||||
def _is_cooldown_required(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model_id: str,
|
||||
exception_status: str | int,
|
||||
exception_str: str | None = None,
|
||||
|
|
@ -320,7 +316,7 @@ def _is_per_user_provider_request(request_kwargs: Mapping[str, object]) -> bool:
|
|||
|
||||
|
||||
def _should_run_cooldown_logic(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str | None,
|
||||
exception_status: str | int,
|
||||
original_exception: Exception,
|
||||
|
|
@ -335,10 +331,10 @@ def _should_run_cooldown_logic(
|
|||
- router.disable_cooldowns is True
|
||||
- deployment is None
|
||||
- _is_cooldown_required() returns False
|
||||
- deployment is in litellm_router_instance.provider_default_deployment_ids
|
||||
- deployment is in ctx.settings().provider_default_deployment_ids
|
||||
- exception_status is not one that should be immediately retried (e.g. 401)
|
||||
"""
|
||||
if deployment is None or litellm_router_instance.get_model_group(id=deployment) is None:
|
||||
if deployment is None or ctx.registry().resolve(deployment, kind="group_of_id") is None:
|
||||
verbose_router_logger.debug(
|
||||
"Should Not Run Cooldown Logic: deployment id is none or model group can't be found."
|
||||
)
|
||||
|
|
@ -369,7 +365,7 @@ def _should_run_cooldown_logic(
|
|||
return False
|
||||
|
||||
if deployment is not None:
|
||||
failed_deployment: Final = litellm_router_instance.get_deployment(model_id=deployment)
|
||||
failed_deployment: Final = ctx.registry().deployment(deployment)
|
||||
if failed_deployment is not None:
|
||||
from litellm.llms.github_copilot.per_user_auth import (
|
||||
github_copilot_per_user_credential_name,
|
||||
|
|
@ -389,7 +385,7 @@ def _should_run_cooldown_logic(
|
|||
verbose_router_logger.debug("Should Not Run Cooldown Logic: time_to_cooldown is effectively 0")
|
||||
return False
|
||||
|
||||
if litellm_router_instance.disable_cooldowns:
|
||||
if ctx.settings().disable_cooldowns:
|
||||
verbose_router_logger.debug("Should Not Run Cooldown Logic: disable_cooldowns is True")
|
||||
return False
|
||||
|
||||
|
|
@ -398,19 +394,19 @@ def _should_run_cooldown_logic(
|
|||
return False
|
||||
|
||||
if not _is_cooldown_required(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
model_id=deployment,
|
||||
exception_status=exception_status,
|
||||
exception_str=str(original_exception),
|
||||
) and not _has_explicit_allowed_fails_policy_for_exception(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment=deployment,
|
||||
original_exception=original_exception,
|
||||
):
|
||||
verbose_router_logger.debug("Should Not Run Cooldown Logic: _is_cooldown_required returned False")
|
||||
return False
|
||||
|
||||
if deployment in litellm_router_instance.provider_default_deployment_ids:
|
||||
if deployment in ctx.settings().provider_default_deployment_ids:
|
||||
verbose_router_logger.debug("Should Not Run Cooldown Logic: deployment is in provider_default_deployment_ids")
|
||||
return False
|
||||
|
||||
|
|
@ -448,10 +444,10 @@ def _fallback_entries(configured: object) -> tuple[object, ...]:
|
|||
return ()
|
||||
|
||||
|
||||
def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool:
|
||||
def _has_fallback_group(ctx: "RoutingContext", requested_model_group: str | None) -> bool:
|
||||
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
|
||||
|
||||
configured: Final[object] = getattr(litellm_router_instance, "fallbacks", None)
|
||||
configured: Final[object] = ctx.settings().fallbacks
|
||||
if requested_model_group is None or not configured:
|
||||
return False
|
||||
chains: Final = tuple(
|
||||
|
|
@ -462,7 +458,7 @@ def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_
|
|||
|
||||
|
||||
def _should_cooldown_deployment(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str,
|
||||
exception_status: str | int,
|
||||
original_exception: Exception,
|
||||
|
|
@ -486,31 +482,29 @@ def _should_cooldown_deployment(
|
|||
|
||||
- v1 logic (Legacy): if allowed fails or allowed fail policy set, coolsdown if num fails in this minute > allowed fails
|
||||
"""
|
||||
model_group: Final = litellm_router_instance.get_model_group(id=deployment)
|
||||
model_group: Final = ctx.registry().resolve(deployment, kind="group_of_id")
|
||||
is_single_deployment_model_group = False
|
||||
if model_group is not None and len(model_group) == 1:
|
||||
is_single_deployment_model_group = not (
|
||||
litellm_router_instance.routing_group_has_alternatives(requested_model_group)
|
||||
or litellm_router_instance.team_model_has_alternatives(deployment)
|
||||
ctx.registry().resolve(requested_model_group, kind="routing_group_alternatives")
|
||||
or ctx.registry().resolve(deployment, kind="team_model_alternatives")
|
||||
)
|
||||
is_lone_deployment: Final = is_single_deployment_model_group and not _has_fallback_group(
|
||||
litellm_router_instance, requested_model_group
|
||||
)
|
||||
is_lone_deployment: Final = is_single_deployment_model_group and not _has_fallback_group(ctx, requested_model_group)
|
||||
|
||||
exception_status_int: Final = cast_exception_status_to_int(exception_status)
|
||||
dep_policy, dep_allowed_fails = _get_deployment_cooldown_policy(litellm_router_instance, deployment)
|
||||
dep_policy, dep_allowed_fails = _get_deployment_cooldown_policy(ctx, deployment)
|
||||
if (
|
||||
is_single_deployment_model_group
|
||||
and exception_status_int == 402
|
||||
and _resolve_allowed_fails_from_policy(dep_policy, original_exception) is None
|
||||
and litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None
|
||||
and ctx.settings().allowed_fails_for(original_exception) is None
|
||||
):
|
||||
return False
|
||||
|
||||
## CHECK DEPLOYMENT-LEVEL POLICY FIRST (overrides router-level)
|
||||
if dep_policy is not None or dep_allowed_fails is not None:
|
||||
return _should_cooldown_based_on_deployment_policy(
|
||||
litellm_router_instance,
|
||||
ctx,
|
||||
deployment,
|
||||
original_exception,
|
||||
dep_policy,
|
||||
|
|
@ -519,16 +513,11 @@ def _should_cooldown_deployment(
|
|||
)
|
||||
|
||||
## BASE CASE - single deployment
|
||||
if (
|
||||
litellm_router_instance.allowed_fails_policy is None
|
||||
and _is_allowed_fails_set_on_router(litellm_router_instance=litellm_router_instance) is False
|
||||
):
|
||||
if ctx.settings().allowed_fails_policy is None and _is_allowed_fails_set_on_router(ctx=ctx) is False:
|
||||
num_successes_this_minute: Final = get_deployment_successes_for_current_minute(
|
||||
litellm_router_instance=litellm_router_instance, deployment_id=deployment
|
||||
)
|
||||
num_fails_this_minute: Final = get_deployment_failures_for_current_minute(
|
||||
litellm_router_instance=litellm_router_instance, deployment_id=deployment
|
||||
ctx=ctx, deployment_id=deployment
|
||||
)
|
||||
num_fails_this_minute: Final = get_deployment_failures_for_current_minute(ctx=ctx, deployment_id=deployment)
|
||||
|
||||
total_requests_this_minute: Final = num_successes_this_minute + num_fails_this_minute
|
||||
percent_fails = 0.0
|
||||
|
|
@ -560,10 +549,10 @@ def _should_cooldown_deployment(
|
|||
|
||||
return False
|
||||
else:
|
||||
if is_lone_deployment and litellm_router_instance.get_allowed_fails_from_policy(original_exception) is None:
|
||||
if is_lone_deployment and ctx.settings().allowed_fails_for(original_exception) is None:
|
||||
return False
|
||||
return should_cooldown_based_on_allowed_fails_policy(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment=deployment,
|
||||
original_exception=original_exception,
|
||||
)
|
||||
|
|
@ -571,8 +560,9 @@ def _should_cooldown_deployment(
|
|||
return False
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def set_cooldown_deployments(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
original_exception: Exception,
|
||||
exception_status: str | int,
|
||||
deployment: str | None = None,
|
||||
|
|
@ -595,7 +585,7 @@ def set_cooldown_deployments(
|
|||
|
||||
if (
|
||||
_should_run_cooldown_logic(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment=deployment,
|
||||
exception_status=exception_status,
|
||||
original_exception=original_exception,
|
||||
|
|
@ -612,13 +602,13 @@ def set_cooldown_deployments(
|
|||
verbose_router_logger.debug("Attempting to add %s to cooldown list", deployment)
|
||||
|
||||
if _should_cooldown_deployment(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment=deployment,
|
||||
exception_status=exception_status,
|
||||
original_exception=original_exception,
|
||||
requested_model_group=requested_model_group,
|
||||
):
|
||||
litellm_router_instance.cooldown_cache.add_deployment_to_cooldown(
|
||||
ctx.health().cooldowns.add_deployment_to_cooldown(
|
||||
model_id=deployment,
|
||||
original_exception=original_exception,
|
||||
exception_status=exception_status_int,
|
||||
|
|
@ -628,7 +618,7 @@ def set_cooldown_deployments(
|
|||
# Trigger cooldown callback handler
|
||||
asyncio.create_task(
|
||||
router_cooldown_event_callback(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
deployment_id=deployment,
|
||||
exception_status=exception_status,
|
||||
cooldown_time=time_to_cooldown,
|
||||
|
|
@ -641,15 +631,16 @@ def set_cooldown_deployments(
|
|||
_set_cooldown_deployments = set_cooldown_deployments
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def async_get_cooldown_deployments(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
parent_otel_span: Span | None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Async implementation of 'get_cooldown_deployments'
|
||||
"""
|
||||
model_ids: Final = litellm_router_instance.get_model_ids()
|
||||
cooldown_models: Final = await litellm_router_instance.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids: Final = ctx.registry().names(kind="ids")
|
||||
cooldown_models: Final = await ctx.health().cooldowns.async_get_active_cooldowns(
|
||||
model_ids=model_ids,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
|
@ -670,15 +661,16 @@ async def async_get_cooldown_deployments(
|
|||
_async_get_cooldown_deployments = async_get_cooldown_deployments
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def async_get_cooldown_deployments_with_debug_info(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
parent_otel_span: Span | None,
|
||||
) -> list[tuple[str, CooldownCacheValue]]:
|
||||
"""
|
||||
Async implementation of 'get_cooldown_deployments'
|
||||
"""
|
||||
model_ids: Final = litellm_router_instance.get_model_ids()
|
||||
cooldown_models: Final = await litellm_router_instance.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids: Final = ctx.registry().names(kind="ids")
|
||||
cooldown_models: Final = await ctx.health().cooldowns.async_get_active_cooldowns(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
|
||||
|
|
@ -689,7 +681,8 @@ async def async_get_cooldown_deployments_with_debug_info(
|
|||
_async_get_cooldown_deployments_with_debug_info = async_get_cooldown_deployments_with_debug_info
|
||||
|
||||
|
||||
def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel_span: Span | None) -> list[str]:
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def get_cooldown_deployments(ctx: "RoutingContext", parent_otel_span: Span | None) -> list[str]:
|
||||
"""
|
||||
Get the list of models being cooled down for this minute
|
||||
"""
|
||||
|
|
@ -698,9 +691,9 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel
|
|||
# ----------------------
|
||||
# Return cooldown models
|
||||
# ----------------------
|
||||
model_ids: Final = litellm_router_instance.get_model_ids()
|
||||
model_ids: Final = ctx.registry().names(kind="ids")
|
||||
|
||||
cooldown_models: Final = litellm_router_instance.cooldown_cache.get_active_cooldowns(
|
||||
cooldown_models: Final = ctx.health().cooldowns.get_active_cooldowns(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
|
||||
|
|
@ -719,28 +712,30 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel
|
|||
_get_cooldown_deployments = get_cooldown_deployments
|
||||
|
||||
|
||||
def _deployment_api_base(litellm_router_instance: LitellmRouter, model_id: str) -> str | None:
|
||||
deployment: Final = litellm_router_instance.get_deployment(model_id=model_id)
|
||||
def _deployment_api_base(ctx: "RoutingContext", model_id: str) -> str | None:
|
||||
deployment: Final = ctx.registry().deployment(model_id)
|
||||
api_base: Final = deployment.litellm_params.api_base if deployment else None
|
||||
return None if api_base is None else mask_api_base_credentials(api_base)
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def get_cooldown_deployment_info(
|
||||
litellm_router_instance: LitellmRouter, route_model_ids: Collection[str], cooldown_list: Sequence[str]
|
||||
ctx: "RoutingContext", route_model_ids: Collection[str], cooldown_list: Sequence[str]
|
||||
) -> tuple[CooldownDeploymentInfo, ...]:
|
||||
"""
|
||||
The deployments the router resolved for this request (`route_model_ids`) that are in `cooldown_list`, each
|
||||
with its masked api_base
|
||||
"""
|
||||
return tuple(
|
||||
CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(litellm_router_instance, model_id))
|
||||
CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(ctx, model_id))
|
||||
for model_id in cooldown_list
|
||||
if model_id in route_model_ids
|
||||
)
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def should_cooldown_based_on_allowed_fails_policy(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment: str,
|
||||
original_exception: Exception,
|
||||
allowed_fails_override: int | None = None,
|
||||
|
|
@ -764,29 +759,21 @@ def should_cooldown_based_on_allowed_fails_policy(
|
|||
- True if fails exceed the allowed limit (should cooldown)
|
||||
- False if fails are within the allowed limit (should not cooldown)
|
||||
"""
|
||||
allowed_fails_from_policy: Final = litellm_router_instance.get_allowed_fails_from_policy(
|
||||
exception=original_exception
|
||||
)
|
||||
allowed_fails_from_policy: Final = ctx.settings().allowed_fails_for(original_exception)
|
||||
allowed_fails: Final = (
|
||||
allowed_fails_override
|
||||
if allowed_fails_override is not None
|
||||
else (
|
||||
allowed_fails_from_policy
|
||||
if allowed_fails_from_policy is not None
|
||||
else litellm_router_instance.allowed_fails
|
||||
)
|
||||
else (allowed_fails_from_policy if allowed_fails_from_policy is not None else ctx.settings().allowed_fails)
|
||||
)
|
||||
cooldown_time: Final = (
|
||||
cooldown_time_override
|
||||
if cooldown_time_override is not None
|
||||
else (litellm_router_instance.cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS)
|
||||
else (ctx.settings().cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS)
|
||||
)
|
||||
|
||||
base_key: Final = f"deployment:{deployment}:allowed_fails"
|
||||
cache_key: Final = f"{base_key}:{cache_key_suffix}" if cache_key_suffix else base_key
|
||||
updated_fails: Final = _increment_allowed_fails(
|
||||
cache=litellm_router_instance.cache, cache_key=cache_key, ttl=cooldown_time
|
||||
)
|
||||
updated_fails: Final = _increment_allowed_fails(cache=ctx.health().store, cache_key=cache_key, ttl=cooldown_time)
|
||||
return updated_fails > allowed_fails
|
||||
|
||||
|
||||
|
|
@ -805,7 +792,7 @@ def _increment_allowed_fails(cache: DualCache, cache_key: str, ttl: float) -> in
|
|||
|
||||
|
||||
def _is_allowed_fails_set_on_router(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
) -> bool:
|
||||
"""
|
||||
Check if Router.allowed_fails was passed to the Router or is a Non-default Value
|
||||
|
|
@ -815,11 +802,11 @@ def _is_allowed_fails_set_on_router(
|
|||
- True if Router.allowed_fails is a Non-default Value
|
||||
- False if Router.allowed_fails is None, or is the Default Value and was not passed
|
||||
"""
|
||||
if litellm_router_instance.allowed_fails is None:
|
||||
if ctx.settings().allowed_fails is None:
|
||||
return False
|
||||
if litellm_router_instance.explicit_allowed_fails is not None:
|
||||
if ctx.explicit_allowed_fails is not None:
|
||||
return True
|
||||
if litellm_router_instance.allowed_fails != litellm.allowed_fails:
|
||||
if ctx.settings().allowed_fails != litellm.allowed_fails:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -13,16 +13,17 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
get_fallback_error_info,
|
||||
)
|
||||
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
|
||||
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging, same_attempt_marker
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
get_fallback_error_info,
|
||||
)
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
_first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils
|
||||
cast_exception_status_to_int,
|
||||
|
|
@ -43,11 +44,7 @@ from litellm.types.router import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
# Status codes a generic API call's caller-supplied resource id can trigger on its own
|
||||
# (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health.
|
||||
|
|
@ -57,7 +54,7 @@ _FALLBACK_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
|||
|
||||
|
||||
def _trigger_cooldown_for_failed_deployment(
|
||||
litellm_router: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
kwargs: Mapping[str, object],
|
||||
exception: Exception,
|
||||
model_call_details: Mapping[str, object] = _NO_MODEL_CALL_DETAILS,
|
||||
|
|
@ -128,7 +125,7 @@ def _trigger_cooldown_for_failed_deployment(
|
|||
verbose_router_logger.debug("Cannot trigger cooldown for fallback: no failed_deployment_id on exception")
|
||||
return
|
||||
|
||||
deployment: Final = litellm_router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = ctx.registry().deployment(deployment_id)
|
||||
if is_caller_scoped_auth_failure(deployment, exception_status):
|
||||
verbose_router_logger.debug(
|
||||
"Not triggering cooldown for fallback deployment %s: caller-scoped OAuth authentication failure.",
|
||||
|
|
@ -138,7 +135,7 @@ def _trigger_cooldown_for_failed_deployment(
|
|||
|
||||
# Priority: deployment config > response header > router default, matching
|
||||
# Router.deployment_callback_on_failure's precedence for the primary path.
|
||||
deployment_dict: Final = litellm_router.get_model_info(id=deployment_id)
|
||||
deployment_dict: Final = ctx.registry().deployment_dict(deployment_id)
|
||||
deployment_cooldown: Final = (
|
||||
_first_present(
|
||||
deployment_dict.get("model_info"), deployment_dict.get("litellm_params"), key="cooldown_time"
|
||||
|
|
@ -161,16 +158,16 @@ def _trigger_cooldown_for_failed_deployment(
|
|||
else (
|
||||
header_cooldown
|
||||
if header_cooldown is not None and header_cooldown >= 0
|
||||
else litellm_router.cooldown_time
|
||||
else ctx.settings().cooldown_time
|
||||
)
|
||||
)
|
||||
|
||||
increment_deployment_failures_for_current_minute(
|
||||
litellm_router_instance=litellm_router,
|
||||
ctx=ctx,
|
||||
deployment_id=deployment_id,
|
||||
)
|
||||
set_cooldown_deployments(
|
||||
litellm_router_instance=litellm_router,
|
||||
ctx=ctx,
|
||||
exception_status=exception_status,
|
||||
original_exception=exception,
|
||||
deployment=deployment_id,
|
||||
|
|
@ -697,14 +694,14 @@ def _get_fallback_target_model_group(fallback_entry: str | Mapping[str, object])
|
|||
|
||||
|
||||
async def _is_fallback_target_authorized(
|
||||
litellm_router: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
fallback_entry: str | Mapping[str, object],
|
||||
original_model_group: str,
|
||||
kwargs: Mapping[str, object],
|
||||
*,
|
||||
attempted: AttemptedFallbackTargets,
|
||||
) -> bool:
|
||||
access_check: Final = litellm_router.fallback_access_check
|
||||
access_check: Final = ctx.settings().fallback_access_check
|
||||
target: Final = _get_fallback_target_model_group(fallback_entry)
|
||||
if (
|
||||
access_check is None
|
||||
|
|
@ -713,7 +710,7 @@ async def _is_fallback_target_authorized(
|
|||
or target == get_pre_routing_selection(kwargs)
|
||||
):
|
||||
return True
|
||||
if await access_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
|
||||
if await ctx.fallback_access_allowed(model=target, request_kwargs=kwargs):
|
||||
return True
|
||||
verbose_router_logger.info(
|
||||
"Skipping fallback to model_group = %s: caller is not authorized to call it",
|
||||
|
|
@ -724,18 +721,18 @@ async def _is_fallback_target_authorized(
|
|||
|
||||
|
||||
async def _is_fallback_target_within_budget(
|
||||
litellm_router: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
fallback_entry: str | Mapping[str, object],
|
||||
original_model_group: str,
|
||||
kwargs: Mapping[str, object],
|
||||
*,
|
||||
attempted: AttemptedFallbackTargets,
|
||||
) -> bool:
|
||||
budget_check: Final = litellm_router.fallback_budget_check
|
||||
budget_check: Final = ctx.settings().fallback_budget_check
|
||||
target: Final = _get_fallback_target_model_group(fallback_entry)
|
||||
if budget_check is None or target is None or target == original_model_group:
|
||||
return True
|
||||
if await budget_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
|
||||
if await ctx.fallback_budget_allowed(model=target, request_kwargs=kwargs):
|
||||
return True
|
||||
verbose_router_logger.info(
|
||||
"Skipping fallback to model_group = %s: caller is over budget",
|
||||
|
|
@ -779,9 +776,10 @@ def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
|
|||
return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router", new="litellm_routing_ctx")
|
||||
async def run_async_fallback(
|
||||
*args: object,
|
||||
litellm_router: LitellmRouter,
|
||||
litellm_routing_ctx: "RoutingContext",
|
||||
fallback_model_group: list[str],
|
||||
original_model_group: str,
|
||||
original_exception: Exception,
|
||||
|
|
@ -798,7 +796,7 @@ async def run_async_fallback(
|
|||
If all fallback model groups fail, it raises the most recent exception.
|
||||
|
||||
Args:
|
||||
litellm_router: The litellm router instance.
|
||||
litellm_routing_ctx: The routing context of the router running the fallbacks.
|
||||
*args: Positional arguments.
|
||||
fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
|
||||
original_model_group: The original model group. example: "gpt-3.5-turbo"
|
||||
|
|
@ -817,6 +815,7 @@ async def run_async_fallback(
|
|||
Raises:
|
||||
The most recent exception if all fallback model groups fail.
|
||||
"""
|
||||
ctx: Final = litellm_routing_ctx
|
||||
|
||||
### BASE CASE ### MAX FALLBACK DEPTH REACHED
|
||||
if fallback_depth >= max_fallbacks:
|
||||
|
|
@ -853,13 +852,9 @@ async def run_async_fallback(
|
|||
original_model_group,
|
||||
)
|
||||
continue
|
||||
if not await _is_fallback_target_authorized(
|
||||
litellm_router, mg, original_model_group, kwargs, attempted=attempted
|
||||
):
|
||||
if not await _is_fallback_target_authorized(ctx, mg, original_model_group, kwargs, attempted=attempted):
|
||||
continue
|
||||
if not await _is_fallback_target_within_budget(
|
||||
litellm_router, mg, original_model_group, kwargs, attempted=attempted
|
||||
):
|
||||
if not await _is_fallback_target_within_budget(ctx, mg, original_model_group, kwargs, attempted=attempted):
|
||||
continue
|
||||
attempt_key = fallback_attempt_key(mg)
|
||||
if attempt_key is not None:
|
||||
|
|
@ -872,7 +867,7 @@ async def run_async_fallback(
|
|||
attempted.record(attempt_key)
|
||||
try:
|
||||
# LOGGING
|
||||
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
|
||||
kwargs = ctx.log_retry(kwargs=kwargs, e=original_exception)
|
||||
verbose_router_logger.info("Falling back to model_group = %s", mask_sensitive_structure(mg))
|
||||
kwargs.pop("_target_order", None)
|
||||
if isinstance(mg, str):
|
||||
|
|
@ -894,7 +889,7 @@ async def run_async_fallback(
|
|||
if include_fallback_errors:
|
||||
kwargs["include_fallback_errors"] = include_fallback_errors
|
||||
carry_superseded_attempt_usage(_FALLBACK_KWARGS_ADAPTER.validate_python(kwargs))
|
||||
response = await litellm_router.async_function_with_fallbacks(*args, **kwargs)
|
||||
response = await ctx.with_fallbacks(*args, **kwargs)
|
||||
verbose_router_logger.info("Successful fallback b/w models.")
|
||||
response = add_fallback_headers_to_response(
|
||||
response=response,
|
||||
|
|
@ -922,7 +917,7 @@ async def run_async_fallback(
|
|||
)
|
||||
) is not None:
|
||||
_trigger_cooldown_for_failed_deployment(
|
||||
litellm_router=litellm_router,
|
||||
ctx=ctx,
|
||||
kwargs=kwargs,
|
||||
exception=e,
|
||||
model_call_details=failure_callbacks_blocked_logging_obj.model_call_details,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
from litellm._logging import redact_secrets, verbose_router_logger
|
||||
from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
async_get_cooldown_deployments_with_debug_info,
|
||||
get_cooldown_deployment_info,
|
||||
|
|
@ -13,17 +14,16 @@ from litellm.types.router import RouterRateLimitError
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.router import Router as _Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
LitellmRouter = _Router
|
||||
Span = _Span
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
Span = Any
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def send_llm_exception_alert(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
request_kwargs: dict,
|
||||
error_traceback_str: str,
|
||||
original_exception,
|
||||
|
|
@ -33,19 +33,14 @@ async def send_llm_exception_alert(
|
|||
Sends a Slack / MS Teams alert for the LLM API call failure. Only if router.slack_alerting_logger is set.
|
||||
|
||||
Parameters:
|
||||
litellm_router_instance (_Router): The LitellmRouter instance.
|
||||
ctx (RoutingContext): The router's routing context.
|
||||
original_exception (Any): The original exception that occurred.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
if litellm_router_instance is None:
|
||||
return
|
||||
|
||||
if not hasattr(litellm_router_instance, "slack_alerting_logger"):
|
||||
return
|
||||
|
||||
if litellm_router_instance.slack_alerting_logger is None:
|
||||
slack_alerting_logger: Final = ctx.observers().slack_alerting_logger
|
||||
if slack_alerting_logger is None:
|
||||
return
|
||||
|
||||
if "proxy_server_request" in request_kwargs:
|
||||
|
|
@ -62,7 +57,7 @@ async def send_llm_exception_alert(
|
|||
# Redact secrets before sending to external service (Slack / MS Teams)
|
||||
exception_str = redact_secrets(exception_str)
|
||||
|
||||
await litellm_router_instance.slack_alerting_logger.send_alert(
|
||||
await slack_alerting_logger.send_alert(
|
||||
message=f"LLM API call failed: `{exception_str}`",
|
||||
level="High",
|
||||
alert_type=AlertType.llm_exceptions,
|
||||
|
|
@ -70,8 +65,9 @@ async def send_llm_exception_alert(
|
|||
)
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def async_raise_no_deployment_exception(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
model: str,
|
||||
parent_otel_span: Span | None,
|
||||
route_model_ids: Collection[str] | None = None,
|
||||
|
|
@ -80,12 +76,12 @@ async def async_raise_no_deployment_exception(
|
|||
Raises a RouterRateLimitError if no deployment is found for the given model.
|
||||
"""
|
||||
verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model)
|
||||
model_ids: Final = litellm_router_instance.get_model_ids(model_name=model)
|
||||
_cooldown_time: Final = litellm_router_instance.cooldown_cache.get_min_cooldown(
|
||||
model_ids: Final = ctx.registry().names(model, kind="ids")
|
||||
_cooldown_time: Final = ctx.health().cooldowns.get_min_cooldown(
|
||||
model_ids=model_ids, parent_otel_span=parent_otel_span
|
||||
)
|
||||
_cooldown_list: Final = await async_get_cooldown_deployments_with_debug_info(
|
||||
litellm_router_instance=litellm_router_instance,
|
||||
ctx=ctx,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
verbose_router_logger.info(
|
||||
|
|
@ -96,12 +92,10 @@ async def async_raise_no_deployment_exception(
|
|||
return RouterRateLimitError(
|
||||
model=model,
|
||||
cooldown_time=_cooldown_time,
|
||||
enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
|
||||
enable_pre_call_checks=ctx.settings().enable_pre_call_checks,
|
||||
cooldown_list=cooldown_list_ids,
|
||||
model_ids=model_ids,
|
||||
cooldown_deployments=(
|
||||
None
|
||||
if route_model_ids is None
|
||||
else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids)
|
||||
None if route_model_ids is None else get_cooldown_deployment_info(ctx, route_model_ids, cooldown_list_ids)
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,8 +14,7 @@ from litellm import verbose_logger
|
|||
from litellm._internal_context import with_service_target
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
|
||||
HEALTH_CHECKS_TARGET: Final = "health_checks"
|
||||
from litellm.constants import HEALTH_CHECKS_TARGET
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
|
||||
from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
||||
from litellm.utils import supports_assistant_prefill
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ from litellm.caching.affinity_cache import (
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
|
|
|||
|
|
@ -50,14 +50,15 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
encrypted_content_of_block,
|
||||
strip_encrypted_reasoning_from_messages,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
|
||||
from litellm.router_deprecated import RenamedRouterAttribute, renamed_router_param
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.utils import get_order_filtered_deployments
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
|
@ -74,14 +75,17 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
per-model group ``model_group_affinity_config``.
|
||||
"""
|
||||
|
||||
router: Final = RenamedRouterAttribute() # deprecated: the router attribute before ``ctx``
|
||||
|
||||
@renamed_router_param("router")
|
||||
def __init__(
|
||||
self,
|
||||
router: Optional["Router"] = None,
|
||||
ctx: Optional["RoutingContext"] = None,
|
||||
enable_global_affinity: bool = True,
|
||||
model_group_affinity_config: dict[str, list[str]] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.router = router
|
||||
self.ctx = ctx
|
||||
self.enable_global_affinity = enable_global_affinity
|
||||
self.model_group_affinity_config: dict[str, list[str]] = model_group_affinity_config or {}
|
||||
|
||||
|
|
@ -254,9 +258,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
deployment's ``(api_base, api_key)``, alongside the originating
|
||||
deployment object (or ``None`` if it was removed / router unavailable).
|
||||
"""
|
||||
if self.router is None:
|
||||
if self.ctx is None:
|
||||
return [], None
|
||||
originating: Final = self.router.get_deployment(model_id=model_id)
|
||||
originating: Final = self.ctx.registry().deployment(model_id)
|
||||
if originating is None:
|
||||
return [], None
|
||||
boundary: Final = self._encryption_boundary_key(originating.litellm_params.model_dump(exclude_none=True))
|
||||
|
|
@ -268,8 +272,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
return matches, originating
|
||||
|
||||
@staticmethod
|
||||
@renamed_router_param("router")
|
||||
def strip_reasoning_the_targets_cannot_decrypt(
|
||||
router: "Router | None",
|
||||
ctx: "RoutingContext | None",
|
||||
request_input: object,
|
||||
anthropic_messages: object,
|
||||
target_deployments: Sequence[Mapping[str, object]],
|
||||
|
|
@ -302,9 +307,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
return False
|
||||
if origin_model_id in target_ids:
|
||||
return True
|
||||
if router is None:
|
||||
if ctx is None:
|
||||
return False
|
||||
origin: Final = router.get_deployment(model_id=origin_model_id)
|
||||
origin: Final = ctx.registry().deployment(origin_model_id)
|
||||
origin_boundary: Final = (
|
||||
EncryptedContentAffinityCheck._encryption_boundary_key(
|
||||
origin.litellm_params.model_dump(exclude_none=True)
|
||||
|
|
@ -419,7 +424,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
)
|
||||
request_kwargs["_encrypted_content_affinity_pinned"] = True
|
||||
self.strip_reasoning_the_targets_cannot_decrypt(
|
||||
self.router, request_input, anthropic_messages, (deployment,), unmarked_origin=model_id
|
||||
self.ctx, request_input, anthropic_messages, (deployment,), unmarked_origin=model_id
|
||||
)
|
||||
return [deployment]
|
||||
|
||||
|
|
@ -437,7 +442,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
)
|
||||
request_kwargs["_encrypted_content_affinity_pinned"] = True
|
||||
self.strip_reasoning_the_targets_cannot_decrypt(
|
||||
self.router, request_input, anthropic_messages, boundary_matches, unmarked_origin=model_id
|
||||
self.ctx, request_input, anthropic_messages, boundary_matches, unmarked_origin=model_id
|
||||
)
|
||||
return boundary_matches
|
||||
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
AnthropicCacheControlHook,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
from litellm.types.utils import CallTypes, StandardLoggingPayload
|
||||
from litellm.utils import (
|
||||
|
|
|
|||
|
|
@ -10,16 +10,13 @@ get_deployment_successes_for_current_minute
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
LitellmRouter = _Router
|
||||
else:
|
||||
LitellmRouter = Any
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
_METADATA_CHANNELS: Final = ("litellm_metadata", "metadata")
|
||||
|
||||
|
|
@ -42,15 +39,16 @@ def get_counted_usage_tokens(litellm_params: Mapping[str, object]) -> int | None
|
|||
return counted if isinstance(counted, int) and not isinstance(counted, bool) else None
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def increment_deployment_successes_for_current_minute(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment_id: str,
|
||||
) -> str:
|
||||
"""
|
||||
In-Memory: Increments the number of successes for the current minute for a deployment_id
|
||||
"""
|
||||
key: Final = f"{deployment_id}:successes"
|
||||
litellm_router_instance.cache.increment_cache(
|
||||
ctx.health().store.increment_cache(
|
||||
local_only=True,
|
||||
key=key,
|
||||
value=1,
|
||||
|
|
@ -59,15 +57,16 @@ def increment_deployment_successes_for_current_minute(
|
|||
return key
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def increment_deployment_failures_for_current_minute(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment_id: str,
|
||||
):
|
||||
"""
|
||||
In-Memory: Increments the number of failures for the current minute for a deployment_id
|
||||
"""
|
||||
key: Final = f"{deployment_id}:fails"
|
||||
litellm_router_instance.cache.increment_cache(
|
||||
ctx.health().store.increment_cache(
|
||||
local_only=True,
|
||||
key=key,
|
||||
value=1,
|
||||
|
|
@ -75,8 +74,9 @@ def increment_deployment_failures_for_current_minute(
|
|||
)
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def get_deployment_successes_for_current_minute(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment_id: str,
|
||||
) -> int:
|
||||
"""
|
||||
|
|
@ -86,7 +86,7 @@ def get_deployment_successes_for_current_minute(
|
|||
"""
|
||||
key: Final = f"{deployment_id}:successes"
|
||||
return (
|
||||
litellm_router_instance.cache.get_cache(
|
||||
ctx.health().store.get_cache(
|
||||
local_only=True,
|
||||
key=key,
|
||||
)
|
||||
|
|
@ -94,8 +94,9 @@ def get_deployment_successes_for_current_minute(
|
|||
)
|
||||
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def get_deployment_failures_for_current_minute(
|
||||
litellm_router_instance: LitellmRouter,
|
||||
ctx: "RoutingContext",
|
||||
deployment_id: str,
|
||||
) -> int:
|
||||
"""
|
||||
|
|
@ -105,7 +106,7 @@ def get_deployment_failures_for_current_minute(
|
|||
"""
|
||||
key: Final = f"{deployment_id}:fails"
|
||||
return (
|
||||
litellm_router_instance.cache.get_cache(
|
||||
ctx.health().store.get_cache(
|
||||
local_only=True,
|
||||
key=key,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,13 +21,14 @@ from litellm._internal_context import service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.redis_batch import BatchResult, active_request_redis_batches
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2, PrefetchedUsage
|
||||
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
|
||||
ROUTER_COOLDOWNS_USAGE_TARGET: Final = "router_cooldowns_usage"
|
||||
|
|
@ -93,23 +94,24 @@ class RoutingPrefetch:
|
|||
raise
|
||||
|
||||
@staticmethod
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
def arm(
|
||||
litellm_router_instance: "Router",
|
||||
ctx: "RoutingContext",
|
||||
usage_selector: LowestTPMLoggingHandler_v2 | None,
|
||||
deployments: list,
|
||||
) -> None:
|
||||
request: Final = active_request_redis_batches()
|
||||
redis_cache: Final = litellm_router_instance.cache.redis_cache
|
||||
redis_cache: Final = ctx.health().store.redis_cache
|
||||
if request is None or redis_cache is None or _PREFETCH_SLOT in request.prefetched:
|
||||
return
|
||||
cooldown_keys: Final = tuple(
|
||||
CooldownCache.get_cooldown_cache_key(model_id) for model_id in litellm_router_instance.get_model_ids()
|
||||
CooldownCache.get_cooldown_cache_key(model_id) for model_id in ctx.registry().names(kind="ids")
|
||||
)
|
||||
usage_keys: Final = (
|
||||
() if usage_selector is None else tuple(itertools.chain(*usage_selector.usage_counter_keys(deployments)))
|
||||
)
|
||||
keys: Final = (*cooldown_keys, *usage_keys)
|
||||
cooldown_store: Final = litellm_router_instance.cooldown_cache.cooldown_store
|
||||
cooldown_store: Final = ctx.health().cooldowns.cooldown_store
|
||||
cooldown_due, cooldown_previous = cooldown_store.reserve_redis_batch_reads(cooldown_keys)
|
||||
usage_cache: Final = None if usage_selector is None else usage_selector.router_cache
|
||||
usage_reservation: Final = None if usage_cache is None else usage_cache.reserve_redis_batch_reads(usage_keys)
|
||||
|
|
@ -184,9 +186,10 @@ class RoutingReadBatch:
|
|||
return RoutingReadBatch(usage_selector=selector)
|
||||
return RoutingReadBatch(usage_selector=None) if RoutingPrefetch.armed() else None
|
||||
|
||||
@renamed_router_param("litellm_router_instance")
|
||||
async def async_get_cooldown_deployments(
|
||||
self,
|
||||
litellm_router_instance: "Router",
|
||||
ctx: "RoutingContext",
|
||||
healthy_deployments: list,
|
||||
parent_otel_span: "Span | None",
|
||||
) -> list[str]:
|
||||
|
|
@ -194,14 +197,14 @@ class RoutingReadBatch:
|
|||
`_async_get_cooldown_deployments`, with the strategy's tpm/rpm counters for
|
||||
`healthy_deployments` fetched in the same MGET and kept as `prefetched_usage`.
|
||||
"""
|
||||
model_ids: Final = litellm_router_instance.get_model_ids()
|
||||
model_ids: Final = ctx.registry().names(kind="ids")
|
||||
cooldown_keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
|
||||
selector: Final = self.usage_selector
|
||||
usage_keys: Final = (
|
||||
() if selector is None else tuple(itertools.chain(*selector.usage_counter_keys(healthy_deployments)))
|
||||
)
|
||||
reads: Final = (
|
||||
(litellm_router_instance.cooldown_cache.cooldown_store, cooldown_keys),
|
||||
(ctx.health().cooldowns.cooldown_store, cooldown_keys),
|
||||
*(() if selector is None else ((selector.router_cache, list(usage_keys)),)),
|
||||
)
|
||||
with service_target(_routing_read_target(cooldown_keys, usage_keys)):
|
||||
|
|
@ -216,9 +219,7 @@ class RoutingReadBatch:
|
|||
values=None if usage_values is None else MappingProxyType(dict(zip(usage_keys, usage_values))),
|
||||
)
|
||||
|
||||
cooldown_models: Final = litellm_router_instance.cooldown_cache.active_cooldowns_from_results(
|
||||
model_ids, cooldown_results
|
||||
)
|
||||
cooldown_models: Final = ctx.health().cooldowns.active_cooldowns_from_results(model_ids, cooldown_results)
|
||||
verbose_router_logger.debug("retrieve cooldown models: %s", cooldown_models)
|
||||
return [model_id for model_id, _ in cooldown_models]
|
||||
|
||||
|
|
|
|||
|
|
@ -10,16 +10,15 @@ import traceback
|
|||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Protocol
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router_api import ChangeSet
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict
|
||||
|
||||
|
||||
class _SearchToolsRouter(Protocol):
|
||||
"""The one router attribute the search-tool helpers read and replace."""
|
||||
|
||||
search_tools: list[SearchToolTypedDict]
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
|
||||
class SearchAPIRouter:
|
||||
|
|
@ -52,14 +51,15 @@ class SearchAPIRouter:
|
|||
return resolved_api_key, resolved_api_base
|
||||
|
||||
@staticmethod
|
||||
async def update_router_search_tools(router_instance: _SearchToolsRouter, search_tools: list):
|
||||
@renamed_router_param("router_instance")
|
||||
async def update_router_search_tools(ctx: "RoutingContext", search_tools: list):
|
||||
"""
|
||||
Update the router with search tools from the database.
|
||||
|
||||
This method is called by a cron job to sync search tools from DB to router.
|
||||
|
||||
Args:
|
||||
router_instance: The Router instance to update
|
||||
ctx: The routing context of the router to update
|
||||
search_tools: List of search tool configurations from the database
|
||||
"""
|
||||
try:
|
||||
|
|
@ -78,7 +78,7 @@ class SearchAPIRouter:
|
|||
router_search_tools.append(router_search_tool)
|
||||
|
||||
# Update the router's search_tools list
|
||||
router_instance.search_tools = router_search_tools
|
||||
ctx.apply(ChangeSet(assign={"search_tools": router_search_tools}))
|
||||
|
||||
verbose_router_logger.info("Successfully updated router with %s search tool(s)", len(router_search_tools))
|
||||
|
||||
|
|
@ -87,15 +87,16 @@ class SearchAPIRouter:
|
|||
raise e
|
||||
|
||||
@staticmethod
|
||||
@renamed_router_param("router_instance")
|
||||
def get_matching_search_tools(
|
||||
router_instance: _SearchToolsRouter,
|
||||
ctx: "RoutingContext",
|
||||
search_tool_name: str,
|
||||
) -> list:
|
||||
"""
|
||||
Get all search tools matching the given name.
|
||||
|
||||
Args:
|
||||
router_instance: The Router instance
|
||||
ctx: The routing context of the router
|
||||
search_tool_name: Name of the search tool to find
|
||||
|
||||
Returns:
|
||||
|
|
@ -105,7 +106,7 @@ class SearchAPIRouter:
|
|||
ValueError: If no matching search tools are found
|
||||
"""
|
||||
matching_tools: Final = [
|
||||
tool for tool in router_instance.search_tools if tool.get("search_tool_name") == search_tool_name
|
||||
tool for tool in ctx.settings().search_tools if tool.get("search_tool_name") == search_tool_name
|
||||
]
|
||||
|
||||
if not matching_tools:
|
||||
|
|
@ -114,8 +115,9 @@ class SearchAPIRouter:
|
|||
return matching_tools
|
||||
|
||||
@staticmethod
|
||||
@renamed_router_param("router_instance", new="litellm_routing_ctx")
|
||||
async def async_search_with_fallbacks(
|
||||
router_instance: Any,
|
||||
litellm_routing_ctx: "RoutingContext",
|
||||
original_function: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
|
|
@ -124,13 +126,14 @@ class SearchAPIRouter:
|
|||
Reuses the router's retry/fallback infrastructure.
|
||||
|
||||
Args:
|
||||
router_instance: The Router instance
|
||||
litellm_routing_ctx: The routing context of the router
|
||||
original_function: The original litellm.asearch function
|
||||
**kwargs: Search parameters including search_tool_name, query, etc.
|
||||
|
||||
Returns:
|
||||
SearchResponse from the search API
|
||||
"""
|
||||
ctx: Final = litellm_routing_ctx
|
||||
try:
|
||||
search_tool_name: Final = kwargs.get("search_tool_name", kwargs.get("model"))
|
||||
|
||||
|
|
@ -140,20 +143,20 @@ class SearchAPIRouter:
|
|||
# Set up kwargs for the fallback system
|
||||
kwargs["model"] = search_tool_name # Use model field for compatibility with fallback system
|
||||
kwargs["original_generic_function"] = original_function
|
||||
# Bind router_instance to the helper method using partial
|
||||
# Bind ctx to the helper method using partial
|
||||
kwargs["original_function"] = partial(
|
||||
SearchAPIRouter.async_search_with_fallbacks_helper,
|
||||
router_instance=router_instance,
|
||||
litellm_routing_ctx=ctx,
|
||||
)
|
||||
|
||||
# Update kwargs before fallbacks (for logging, metadata, etc)
|
||||
router_instance._update_kwargs_before_fallbacks(
|
||||
ctx.update_kwargs_before_fallbacks(
|
||||
model=search_tool_name,
|
||||
kwargs=kwargs,
|
||||
metadata_variable_name="litellm_metadata",
|
||||
)
|
||||
|
||||
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in router_instance.search_tools]
|
||||
available_search_tool_names: Final = [tool.get("search_tool_name") for tool in ctx.settings().search_tools]
|
||||
verbose_router_logger.debug(
|
||||
"Inside SearchAPIRouter.async_search_with_fallbacks() - search_tool_name: %s, Available Search Tools: %s",
|
||||
search_tool_name,
|
||||
|
|
@ -161,7 +164,7 @@ class SearchAPIRouter:
|
|||
)
|
||||
|
||||
# Use the existing retry/fallback infrastructure
|
||||
response: Final = await router_instance.async_function_with_fallbacks(**kwargs)
|
||||
response: Final = await ctx.with_fallbacks(**kwargs)
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -169,7 +172,7 @@ class SearchAPIRouter:
|
|||
|
||||
asyncio.create_task(
|
||||
send_llm_exception_alert(
|
||||
litellm_router_instance=router_instance,
|
||||
ctx=ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
|
|
@ -178,8 +181,9 @@ class SearchAPIRouter:
|
|||
raise e
|
||||
|
||||
@staticmethod
|
||||
@renamed_router_param("router_instance", new="litellm_routing_ctx")
|
||||
async def async_search_with_fallbacks_helper(
|
||||
router_instance: _SearchToolsRouter,
|
||||
litellm_routing_ctx: "RoutingContext",
|
||||
model: str,
|
||||
original_generic_function: Callable,
|
||||
**kwargs,
|
||||
|
|
@ -189,7 +193,7 @@ class SearchAPIRouter:
|
|||
Called by async_function_with_fallbacks for each retry attempt.
|
||||
|
||||
Args:
|
||||
router_instance: The Router instance
|
||||
litellm_routing_ctx: The routing context of the router
|
||||
model: The search tool name (passed as model for compatibility)
|
||||
original_generic_function: The original litellm.asearch function
|
||||
**kwargs: Search parameters
|
||||
|
|
@ -197,12 +201,13 @@ class SearchAPIRouter:
|
|||
Returns:
|
||||
SearchResponse from the selected search provider
|
||||
"""
|
||||
ctx: Final = litellm_routing_ctx
|
||||
search_tool_name: Final = model # model field contains the search_tool_name
|
||||
|
||||
try:
|
||||
# Find matching search tools
|
||||
matching_tools: Final = SearchAPIRouter.get_matching_search_tools(
|
||||
router_instance=router_instance,
|
||||
ctx=ctx,
|
||||
search_tool_name=search_tool_name,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ def finalize(
|
|||
update(response, logger, model if isinstance(model, str) else None, kwargs, start_time, end_time)
|
||||
cache_key: Final = logger.model_call_details.get("cache_key")
|
||||
if logger.model_call_details.get("cache_hit") is True and isinstance(cache_key, str):
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
|
||||
hidden: Final = get_hidden_params_dict(response, create=True)
|
||||
hidden.update({"cache_key": cache_key, "cache_hit": True})
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Final, TypeVar
|
||||
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
_add_headers_to_response, # pyright: ignore[reportPrivateUsage] # reuse the proxy's identity-preserving response metadata writer
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ if TYPE_CHECKING:
|
|||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.router_api import AttemptedFallbackTargets
|
||||
from litellm.router_strategy.complexity_router.context_compaction import CompactionState
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
|
||||
from litellm.types.caching import DynamicCacheControl
|
||||
from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.headroom import HeadroomRequestContext
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ from litellm.types.llms.base import LiteLLMBaseModel
|
|||
from litellm.types.router_weights import RouterWeights
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RoutingContext as RouterContext
|
||||
|
||||
from .completion import CompletionRequest
|
||||
from .embedding import EmbeddingRequest
|
||||
|
|
@ -1276,7 +1276,7 @@ class FallbackAccessCheck(Protocol):
|
|||
rejects, so a fallback can never reach a model the caller could not have requested directly.
|
||||
"""
|
||||
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ...
|
||||
|
||||
|
||||
class FallbackBudgetCheck(Protocol):
|
||||
|
|
@ -1289,7 +1289,7 @@ class FallbackBudgetCheck(Protocol):
|
|||
and skips targets it rejects, leaving the free attempt itself untouched.
|
||||
"""
|
||||
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
|
||||
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: "RouterContext") -> bool: ...
|
||||
|
||||
|
||||
class AutoRouterCapabilityLimit(Protocol):
|
||||
|
|
|
|||
|
|
@ -363,6 +363,10 @@ if TYPE_CHECKING:
|
|||
get_llm_provider,
|
||||
is_non_openai_azure_model,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_retry_from_policy import (
|
||||
get_num_retries_from_retry_policy,
|
||||
reset_retry_policy,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
|
|
@ -455,10 +459,6 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.router_utils.get_retry_from_policy import (
|
||||
get_num_retries_from_retry_policy,
|
||||
reset_retry_policy,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_API_ONLY_HEADERS,
|
||||
AnthropicThinkingParam,
|
||||
|
|
@ -5258,7 +5258,7 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c
|
|||
|
||||
|
||||
def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tuple[str, ...]:
|
||||
from litellm.router_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
|
||||
|
||||
return deployment_reasoning_effort_levels(model_info) or ()
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ def percentile(values: Sequence[float], quantile: float) -> float:
|
|||
|
||||
|
||||
def verify(response: object, expected_transport: str) -> None:
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
|
||||
pages: Final = getattr(response, "pages", ())
|
||||
if len(pages) != 1 or getattr(pages[0], "markdown", None) != EXPECTED_MARKDOWN:
|
||||
|
|
|
|||
|
|
@ -198,11 +198,11 @@ def main():
|
|||
router_file = [
|
||||
"./litellm/router.py",
|
||||
"./litellm/litellm_core_utils/batch_utils.py",
|
||||
"./litellm/router_utils/pattern_match_deployments.py",
|
||||
"./litellm/litellm_core_utils/pattern_match_deployments.py",
|
||||
]
|
||||
# router_file = [
|
||||
# "../../litellm/router.py",
|
||||
# "../../litellm/router_utils/pattern_match_deployments.py",
|
||||
# "../../litellm/litellm_core_utils/pattern_match_deployments.py",
|
||||
# "../../litellm/litellm_core_utils/batch_utils.py",
|
||||
# ] ## LOCAL TESTING
|
||||
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
{
|
||||
"_doc": "Router members and litellm.router_utils modules used outside the router. Shrink-only: see tests/code_coverage_tests/router_surface_check.py.",
|
||||
"members_outside": [
|
||||
"_get_client",
|
||||
"a_add_message",
|
||||
"aadapter_completion",
|
||||
"aanthropic_messages",
|
||||
|
|
@ -38,79 +37,14 @@
|
|||
"avector_store_search",
|
||||
"completion",
|
||||
"embedding",
|
||||
"get_routing_group",
|
||||
"get_valid_args",
|
||||
"health",
|
||||
"observe",
|
||||
"priority_header_update",
|
||||
"registry",
|
||||
"retire",
|
||||
"schedule_acompletion",
|
||||
"select",
|
||||
"settings",
|
||||
"team_public_model_names_of"
|
||||
"settings"
|
||||
],
|
||||
"members_router_family": [
|
||||
"_count_pre_call_check_tokens",
|
||||
"_pre_call_checks",
|
||||
"_update_kwargs_before_fallbacks",
|
||||
"acompletion",
|
||||
"adecisions",
|
||||
"aembedding",
|
||||
"allowed_fails",
|
||||
"allowed_fails_policy",
|
||||
"aresponses",
|
||||
"async_function_with_fallbacks",
|
||||
"async_get_healthy_deployments",
|
||||
"cache",
|
||||
"cached_deployment_model_info",
|
||||
"cooldown_cache",
|
||||
"cooldown_time",
|
||||
"default_max_parallel_requests",
|
||||
"deployments_named",
|
||||
"disable_cooldowns",
|
||||
"embedding",
|
||||
"enable_pre_call_checks",
|
||||
"enable_tag_filtering",
|
||||
"explicit_allowed_fails",
|
||||
"fallback_access_check",
|
||||
"fallback_budget_check",
|
||||
"fallbacks",
|
||||
"get_all_deployments",
|
||||
"get_allowed_fails_from_policy",
|
||||
"get_deployment",
|
||||
"get_model_group",
|
||||
"get_model_ids",
|
||||
"get_model_info",
|
||||
"get_model_list",
|
||||
"get_router_model_info",
|
||||
"log_retry",
|
||||
"model_list",
|
||||
"provider_default_deployment_ids",
|
||||
"routing_group_has_alternatives",
|
||||
"routing_strategy",
|
||||
"search_tools",
|
||||
"set_custom_routing_strategy",
|
||||
"slack_alerting_logger",
|
||||
"tag_filtering_match_any",
|
||||
"tag_routing_prefix",
|
||||
"team_model_has_alternatives"
|
||||
],
|
||||
"router_utils_imports_outside": [
|
||||
"litellm.router_utils.access_windows",
|
||||
"litellm.router_utils.add_retry_fallback_headers",
|
||||
"litellm.router_utils.clientside_credential_handler",
|
||||
"litellm.router_utils.common_utils",
|
||||
"litellm.router_utils.cooldown_handlers",
|
||||
"litellm.router_utils.deployment_pricing",
|
||||
"litellm.router_utils.fallback_event_handlers",
|
||||
"litellm.router_utils.get_retry_from_policy",
|
||||
"litellm.router_utils.health_state_cache",
|
||||
"litellm.router_utils.pattern_match_deployments",
|
||||
"litellm.router_utils.ptu_shares",
|
||||
"litellm.router_utils.reasoning_effort_capability",
|
||||
"litellm.router_utils.router_callbacks.track_deployment_metrics",
|
||||
"litellm.router_utils.routing_groups",
|
||||
"litellm.router_utils.search_api_router"
|
||||
]
|
||||
"members_router_family": [],
|
||||
"router_utils_imports_outside": []
|
||||
}
|
||||
|
|
|
|||
|
|
@ -143,6 +143,6 @@ async def test_string_valued_fallbacks_still_cool_down_the_failing_primary(fallb
|
|||
model="primary",
|
||||
messages=[{"role": "user", "content": f"cooldown {uuid.uuid4().hex}"}],
|
||||
)
|
||||
cooled: Final = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None)
|
||||
cooled: Final = await async_get_cooldown_deployments(ctx=router._core_ctx, parent_otel_span=None)
|
||||
|
||||
assert tuple(cooled) == ("primary-deployment",)
|
||||
|
|
|
|||
2
tests/test_litellm_rust/cache/support.py
vendored
2
tests/test_litellm_rust/cache/support.py
vendored
|
|
@ -7,7 +7,7 @@ from typing import Final, Literal
|
|||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.rust_bridge import runtime
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule
|
||||
from litellm.rust_bridge.chat_completions.entrypoints import NATIVE_ACOMPLETION
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from litellm.caching.caching import Cache, CacheMode
|
|||
from litellm.caching.caching_handler import (
|
||||
_PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request
|
||||
)
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
PROXY_VirtualKeyModelMaxBudgetLimiter,
|
||||
|
|
@ -17,7 +18,6 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
|
|||
)
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.rust_bridge import runtime
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule
|
||||
from litellm.rust_bridge.configuration import Rollout
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from litellm._v2.cache import NativeBackend
|
|||
from litellm.caching.caching_handler import (
|
||||
_PENDING_CACHE_WRITES, # pyright: ignore[reportPrivateUsage] # await the existing background cache writer before the next request
|
||||
)
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.types.utils import ModelResponse
|
||||
from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE
|
||||
from tests.test_litellm_rust.responses.data import RESPONSES_MODEL, RESPONSES_RESPONSE, RESPONSES_TEXT
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ _SDK_CONTRACT = textwrap.dedent(
|
|||
|
||||
import litellm
|
||||
from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
PROXY_MaxParallelRequestsHandler_v3,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCache
|
||||
from litellm.router_utils.routing_read_batch import (
|
||||
ROUTER_COOLDOWNS_USAGE_TARGET,
|
||||
|
|
@ -390,7 +391,7 @@ async def test_a_prefetch_with_incomplete_usage_keys_releases_cooldown_reservati
|
|||
)
|
||||
|
||||
with request_redis_batch_scope():
|
||||
RoutingPrefetch.arm(router, router.lowesttpm_logger_v2, router.model_list[:1])
|
||||
RoutingPrefetch.arm(RoutingContext(router), router.lowesttpm_logger_v2, router.model_list[:1])
|
||||
deployment: Final = await router.async_get_available_deployment(
|
||||
model=_MODEL_GROUP, messages=[{"role": "user", "content": "ping"}], request_kwargs={}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -241,21 +241,21 @@ async def test_router_cooldown_event_callback():
|
|||
|
||||
Ensures that the router_cooldown_event_callback function correctly logs the cooldown event to the PrometheusLogger
|
||||
"""
|
||||
# Mock Router instance
|
||||
mock_router = MagicMock()
|
||||
# Mock routing context
|
||||
mock_ctx = MagicMock()
|
||||
mock_deployment = {
|
||||
"litellm_params": {"model": "gpt-5-mini"},
|
||||
"model_name": "gpt-5-mini",
|
||||
"model_info": ModelInfo(id="test-model-id"),
|
||||
}
|
||||
mock_router.get_deployment.return_value = mock_deployment
|
||||
mock_ctx.registry.return_value.deployment.return_value = mock_deployment
|
||||
|
||||
# Create a real PrometheusLogger instance
|
||||
prometheus_logger = CustomPrometheusLogger()
|
||||
litellm.callbacks = [prometheus_logger]
|
||||
|
||||
await router_cooldown_event_callback(
|
||||
litellm_router_instance=mock_router,
|
||||
ctx=mock_ctx,
|
||||
deployment_id="test-deployment",
|
||||
exception_status="429",
|
||||
cooldown_time=60.0,
|
||||
|
|
@ -263,8 +263,8 @@ async def test_router_cooldown_event_callback():
|
|||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Assert that the router's get_deployment method was called
|
||||
mock_router.get_deployment.assert_called_once_with(model_id="test-deployment")
|
||||
# Assert that the router's deployment lookup was called
|
||||
mock_ctx.registry.return_value.deployment.assert_called_once_with("test-deployment")
|
||||
|
||||
print(
|
||||
"prometheus_logger.deployment_complete_outages",
|
||||
|
|
@ -301,21 +301,21 @@ async def test_router_cooldown_event_callback_no_prometheus():
|
|||
|
||||
Ensures that the router_cooldown_event_callback function does not raise an error when no PrometheusLogger is found
|
||||
"""
|
||||
# Mock Router instance
|
||||
mock_router = MagicMock()
|
||||
# Mock routing context
|
||||
mock_ctx = MagicMock()
|
||||
mock_deployment = {
|
||||
"litellm_params": {"model": "gpt-5-mini"},
|
||||
"model_name": "gpt-5-mini",
|
||||
"model_info": ModelInfo(id="test-model-id"),
|
||||
}
|
||||
mock_router.get_deployment.return_value = mock_deployment
|
||||
mock_ctx.registry.return_value.deployment.return_value = mock_deployment
|
||||
|
||||
await router_cooldown_event_callback(
|
||||
litellm_router_instance=mock_router,
|
||||
ctx=mock_ctx,
|
||||
deployment_id="test-deployment",
|
||||
exception_status="429",
|
||||
cooldown_time=60.0,
|
||||
)
|
||||
|
||||
# Assert that the router's get_deployment method was called
|
||||
mock_router.get_deployment.assert_called_once_with(model_id="test-deployment")
|
||||
# Assert that the router's deployment lookup was called
|
||||
mock_ctx.registry.return_value.deployment.assert_called_once_with("test-deployment")
|
||||
|
|
|
|||
|
|
@ -6879,7 +6879,7 @@ def test_rebuilding_a_priced_stream_bills_the_margin_once(monkeypatch: pytest.Mo
|
|||
|
||||
|
||||
def test_chunk_creator_keeps_provider_model_private_across_stream():
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
|
||||
|
|
@ -6940,7 +6940,7 @@ def test_chunk_creator_keeps_provider_model_private_across_stream():
|
|||
def test_assembled_stream_uses_later_provider_model_for_cost(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue