diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ccded64c3a5..547c76587bb 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1153,31 +1153,21 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"No model configured for tier {tier_key} and no default_model set") @staticmethod - def _pick_from_tier_value( - model: str | list[str] | TierTarget, tier_key: str - ) -> str: # mutable-ok: legacy pool inputs remain lists - if isinstance(model, str): - return model + def _pick_from_tier_value(model: str | list[str] | TierTarget, tier_key: str) -> str: pool: Final = tier_pool(model) if not pool: raise ValueError(f"Empty model pool for tier {tier_key}") - return random.choice(pool) + return random.choice(list(pool)) - def _tier_pools(self) -> dict[str, list[str]]: # mutable-ok: adaptive router consumes mutable pools - return { # mutable-ok: router consumers require mutable tier pool mappings - tier: tier_pool(target) for tier, target in self.config.tiers.items() - } + def _tier_pools(self) -> Mapping[str, tuple[str, ...]]: + return MappingProxyType({tier: tier_pool(target) for tier, target in self.config.tiers.items()}) - def _tier_params( - self, tier: ComplexityTier | str - ) -> dict[str, object] | None: # mutable-ok: params are merged into request kwargs + def _tier_params(self, tier: ComplexityTier | str) -> Mapping[str, object] | None: tier_key: Final = tier.value if isinstance(tier, ComplexityTier) else tier target: Final = self.config.tiers.get(tier_key) return target.params or None if isinstance(target, TierTarget) else None - def _params_for_model( - self, model: str - ) -> dict[str, object] | None: # mutable-ok: params are merged into request kwargs + def _params_for_model(self, model: str) -> Mapping[str, object] | None: tier: Final = self._tier_for_model(model) return self._tier_params(tier) if tier is not None else None diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 15d70035893..a4a9b489b29 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -5,13 +5,16 @@ Contains default keyword lists, weights, tier boundaries, and configuration clas All values are configurable via proxy config.yaml. """ +from collections.abc import Mapping from enum import Enum +from types import MappingProxyType from typing import Final, Literal, cast from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from litellm._logging import verbose_router_logger from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS +from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.router import AdaptiveRouterWeights, RoutingPlugin from litellm.types.utils import all_litellm_params @@ -32,6 +35,8 @@ TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( ComplexityTier.REASONING, ) +TIER_STRUCTURAL_KEYS: Final[frozenset[str]] = frozenset({"messages", "input", "stream", "metadata", "litellm_metadata"}) + DEFAULT_TIER_DISTANCE_PENALTY: Final[float] = 0.5 DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE: Final[int] = 3 @@ -243,7 +248,9 @@ DEFAULT_TIER_MODELS: Final[dict[str, str]] = { class TierTarget(BaseModel): - model: str | list[str] # mutable-ok: public config accepts model pools as lists + model: str | list[str] # mutable-ok: pydantic accepts list-valued model pools + reasoning_effort: str | None = None + thinking: AnthropicThinkingParam | None = None model_config = ConfigDict(extra="allow") @@ -265,15 +272,15 @@ class TierTarget(BaseModel): raise ValueError("model must be a string or a list of strings") @property - def params(self) -> dict[str, object]: # mutable-ok: extras are passed through as request kwargs - return cast(dict[str, object], self.__pydantic_extra__ or {}) # cast-ok: pydantic owns extra params + def params(self) -> Mapping[str, object]: + return MappingProxyType(self.model_dump(exclude={"model"}, exclude_none=True)) -def tier_pool(value: str | list[str] | TierTarget) -> list[str]: # mutable-ok: routing pools use list semantics +def tier_pool(value: str | list[str] | TierTarget) -> tuple[str, ...]: if isinstance(value, TierTarget): target_model: Final = value.model - return target_model if isinstance(target_model, list) else [target_model] - return value if isinstance(value, list) else [value] + return (target_model,) if isinstance(target_model, str) else tuple(target_model) + return (value,) if isinstance(value, str) else tuple(value) class ClassifierLLMConfig(BaseModel): @@ -587,6 +594,17 @@ class ComplexityRouterConfig(BaseModel): for tier, target in self.tiers.items(): if isinstance(target, TierTarget): for key in target.params: + if key in TIER_STRUCTURAL_KEYS: + raise ValueError( + f"ComplexityRouter tier {tier} cannot configure structural request key {key!r}" + ) + if key == "thinking_budget": + verbose_router_logger.warning( + "ComplexityRouter tier %s uses thinking_budget; use " + "thinking: {type: enabled, budget_tokens: N} instead", + tier, + ) + continue if key not in known: verbose_router_logger.warning( "ComplexityRouter tier %s has an unrecognized parameter key: %s", @@ -600,11 +618,13 @@ class ComplexityRouterConfig(BaseModel): if not self.adaptive: return self normalized: Final[ - dict[str, str | list[str] | TierTarget] # mutable-ok: pydantic config surface is mutable - ] = { # mutable-ok: pydantic requires normalized tier mappings - tier: target.model_copy(update={"model": tier_pool(target)}) + dict[str, str | list[str] | TierTarget] + ] = { # mutable-ok: pydantic requires normalized mapping + tier: target.model_copy( + update={"model": list(tier_pool(target))} + ) # mutable-ok: pydantic stores pools as lists if isinstance(target, TierTarget) - else tier_pool(target) + else list(tier_pool(target)) # mutable-ok: pydantic stores pools as lists for tier, target in self.tiers.items() } if not any(tier_pool(target) for target in normalized.values()): diff --git a/litellm/types/router.py b/litellm/types/router.py index e44b9a8d1ca..b3d5e7cf265 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +from collections.abc import Mapping from dataclasses import dataclass from typing import Any, Final, Generic, Literal, TypeVar, get_type_hints @@ -819,7 +820,7 @@ class PreRoutingHookResponse(BaseModel): model: str messages: list[dict[str, Any]] | None - params: dict[str, Any] | None = None # mutable-ok: router merges params into request kwargs + params: Mapping[str, object] | None = None routing_decision: StandardLoggingRoutingDecision | None = None diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 7cb402969ab..b9de9161986 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -171,9 +171,7 @@ class TestComplexityRouterInit: ComplexityRouterConfig(tiers={"SIMPLE": tier_value}) def test_tier_target_accepts_pool_and_extra_params(self): - config = ComplexityRouterConfig( - tiers={"SIMPLE": {"model": ["gpt-5", "o3"], "reasoning_effort": "high"}} - ) + config = ComplexityRouterConfig(tiers={"SIMPLE": {"model": ["gpt-5", "o3"], "reasoning_effort": "high"}}) target = config.tiers["SIMPLE"] assert isinstance(target, TierTarget) assert target.model == ["gpt-5", "o3"] @@ -186,6 +184,16 @@ class TestComplexityRouterInit: assert isinstance(config.tiers["SIMPLE"], TierTarget) assert config.tiers["SIMPLE"].params["thinking_level"] == "high" + def test_thinking_budget_warning_explains_structured_syntax(self, caplog): + with caplog.at_level(logging.WARNING, logger=verbose_router_logger.name): + ComplexityRouterConfig(tiers={"SIMPLE": {"model": "gpt-5", "thinking_budget": 1024}}) + assert "thinking: {type: enabled, budget_tokens: N}" in caplog.text + + @pytest.mark.parametrize("structural_key", ["messages", "input", "stream", "metadata", "litellm_metadata"]) + def test_structural_tier_params_are_rejected(self, structural_key): + with pytest.raises(ValidationError, match=structural_key): + ComplexityRouterConfig(tiers={"SIMPLE": {"model": "gpt-5", structural_key: "invalid"}}) + @pytest.mark.asyncio async def test_tier_params_are_applied_with_request_and_alias_precedence(self): router = Router( @@ -4469,7 +4477,6 @@ class TestRoutingDecisionContents: # The score is still recorded, but the cause is what says it did not decide. assert decision["score"] < decision["tier_boundaries"]["complex_reasoning"] - @pytest.mark.asyncio async def test_an_unrenamed_router_writes_no_tier_label(self, complexity_router): """Renaming is opt-in, so a deployment that never renamed must gain no new key. @@ -5924,7 +5931,9 @@ class TestCustomClassifierSystemPrompt: @pytest.mark.asyncio async def test_custom_prompt_is_sent_verbatim_as_the_system_role(self, mock_router_instance, llm_classifier_config): - custom = "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + custom = ( + "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + ) router = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 28fc62d3771..88b1210609f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21222,6 +21222,18 @@ export interface components { /** Index Permissions */ index_permissions: ("read" | "write")[]; }; + /** AnthropicThinkingParam */ + AnthropicThinkingParam: { + /** Budget Tokens */ + budget_tokens?: number; + /** + * Type + * @enum {string} + */ + type?: "enabled" | "adaptive"; + } & { + [key: string]: unknown; + }; /** ApplyGuardrailRequest */ ApplyGuardrailRequest: { /** Entities */ @@ -33263,6 +33275,9 @@ export interface components { TierTarget: { /** Model */ model: string | string[]; + /** Reasoning Effort */ + reasoning_effort?: string | null; + thinking?: components["schemas"]["AnthropicThinkingParam"] | null; } & { [key: string]: unknown; };