mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-23 05:34:41 +00:00
refactor(router): remove the undocumented lar1 routing strategy
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia
parent
090a4c3f24
commit
fc9eefaca5
10 changed files with 14 additions and 770 deletions
|
|
@ -1,45 +0,0 @@
|
|||
model_list:
|
||||
- model_name: agent-router
|
||||
litellm_params:
|
||||
model: ollama/qwen3.5:9b
|
||||
api_base: http://127.0.0.1:11434
|
||||
model_info:
|
||||
id: cloud-smart
|
||||
type: cloud-smart
|
||||
|
||||
- model_name: agent-router
|
||||
litellm_params:
|
||||
model: ollama/phi4-mini:latest
|
||||
api_base: http://127.0.0.1:11434
|
||||
model_info:
|
||||
id: cloud-fast
|
||||
type: cloud-fast
|
||||
|
||||
- model_name: agent-router
|
||||
litellm_params:
|
||||
model: ollama/llama3.2:3b
|
||||
api_base: http://127.0.0.1:11434
|
||||
model_info:
|
||||
id: local
|
||||
type: local
|
||||
|
||||
- model_name: agent-router
|
||||
litellm_params:
|
||||
model: ollama/lfm2.5-thinking:latest
|
||||
api_base: http://127.0.0.1:11434
|
||||
model_info:
|
||||
id: deep
|
||||
type: deep
|
||||
|
||||
router_settings:
|
||||
routing_strategy: lar1
|
||||
routing_strategy_args:
|
||||
confidence_threshold_low: 0.3
|
||||
confidence_threshold_medium: 0.5
|
||||
confidence_threshold_high: 0.7
|
||||
|
||||
general_settings:
|
||||
master_key: sk-lar1-demo
|
||||
|
||||
litellm_settings:
|
||||
set_verbose: true
|
||||
|
|
@ -1254,15 +1254,10 @@ class Router:
|
|||
"""
|
||||
|
||||
### ROUTING SETUP ###
|
||||
if self._normalize_strategy(routing_strategy) == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, routing_strategy_args)
|
||||
else:
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=routing_strategy,
|
||||
routing_strategy_args=routing_strategy_args,
|
||||
)
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=routing_strategy,
|
||||
routing_strategy_args=routing_strategy_args,
|
||||
)
|
||||
self._init_routing_groups(self._routing_groups_input)
|
||||
self._override_selectors: dict[str, RouterStrategySelector | None] = {}
|
||||
self._override_selectors_lock = threading.Lock()
|
||||
|
|
@ -1522,12 +1517,6 @@ class Router:
|
|||
cache, not on the selector.
|
||||
"""
|
||||
strategy: Final = self._normalize_strategy(self.routing_strategy)
|
||||
if strategy == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, self.routing_strategy_args)
|
||||
return
|
||||
|
||||
attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "")
|
||||
current: Final = getattr(self, attr, None) if attr is not None else None
|
||||
if attr is None or current is None:
|
||||
|
|
@ -1753,7 +1742,7 @@ class Router:
|
|||
from key/team `router_settings`) out of the request kwargs.
|
||||
|
||||
Only strategies with a per-request-capable selector are honored;
|
||||
anything else (unknown strings, `lar1`, `provider-budget-routing`) is
|
||||
anything else (unknown strings, `provider-budget-routing`) is
|
||||
ignored with a warning so a bad value stored on a key or team can
|
||||
never take down that caller's traffic.
|
||||
"""
|
||||
|
|
@ -12591,16 +12580,6 @@ class Router:
|
|||
]
|
||||
return _settings_to_return
|
||||
|
||||
def _switch_routing_strategy(self, routing_strategy: str | None, kwargs: Mapping[str, object]) -> None:
|
||||
if routing_strategy == "lar1":
|
||||
from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy
|
||||
|
||||
apply_lar1_routing_strategy(self, kwargs.get("routing_strategy_args"))
|
||||
return
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=routing_strategy, routing_strategy_args=kwargs.get("routing_strategy_args", {})
|
||||
)
|
||||
|
||||
def update_settings(self, **kwargs):
|
||||
"""
|
||||
Update the router settings.
|
||||
|
|
@ -12638,7 +12617,10 @@ class Router:
|
|||
if var == "routing_strategy":
|
||||
value = self._normalize_strategy(value)
|
||||
if _existing_router_settings["routing_strategy"] != value:
|
||||
self._switch_routing_strategy(value, kwargs)
|
||||
self.routing_strategy_init(
|
||||
routing_strategy=value,
|
||||
routing_strategy_args=kwargs.get("routing_strategy_args", {}),
|
||||
)
|
||||
rebuild_routing_groups = True
|
||||
elif var == "routing_strategy_args":
|
||||
routing_args_updated = value != self.routing_strategy_args
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
"""
|
||||
LAR-1 Semantic Routing Strategy
|
||||
|
||||
Routes requests based on agent confidence level (LAR-1 protocol).
|
||||
Thresholds are configurable via routing_strategy_args in router config.
|
||||
|
||||
LAR-1 metadata passed via request_kwargs["metadata"]["lar1"]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router import CustomRoutingStrategyBase
|
||||
from litellm.types.lar1 import LAR1Metadata, LAR1Time
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
DEFAULT_THRESHOLDS: Final[dict[str, float]] = {"low": 0.3, "medium": 0.5, "high": 0.7}
|
||||
|
||||
|
||||
def _coerce_threshold(value: object, default: float) -> float:
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
return default
|
||||
|
||||
|
||||
def lar1_thresholds_from_args(
|
||||
routing_strategy_args: Mapping[str, object] | None = None,
|
||||
) -> dict[str, float]:
|
||||
args: Final = routing_strategy_args or {}
|
||||
return {
|
||||
"low": _coerce_threshold(args.get("confidence_threshold_low"), DEFAULT_THRESHOLDS["low"]),
|
||||
"medium": _coerce_threshold(args.get("confidence_threshold_medium"), DEFAULT_THRESHOLDS["medium"]),
|
||||
"high": _coerce_threshold(args.get("confidence_threshold_high"), DEFAULT_THRESHOLDS["high"]),
|
||||
}
|
||||
|
||||
|
||||
def apply_lar1_routing_strategy(
|
||||
router: Router,
|
||||
routing_strategy_args: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
strategy: Final = LAR1RoutingStrategy(
|
||||
router_instance=router,
|
||||
thresholds=lar1_thresholds_from_args(routing_strategy_args),
|
||||
)
|
||||
router.routing_strategy = "lar1"
|
||||
router.set_custom_routing_strategy(strategy)
|
||||
|
||||
|
||||
def _normalize_thresholds(thresholds: dict[str, float] | None) -> dict[str, float]:
|
||||
merged: Final = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
|
||||
low: Final = merged["low"]
|
||||
medium: Final = merged["medium"]
|
||||
high: Final = merged["high"]
|
||||
if not (0 < low < medium < high < 1):
|
||||
raise ValueError(
|
||||
f"LAR-1 thresholds must satisfy 0 < low < medium < high < 1, got low={low}, medium={medium}, high={high}"
|
||||
)
|
||||
return merged
|
||||
|
||||
|
||||
def _parse_lar1_metadata(request_kwargs: dict) -> LAR1Metadata:
|
||||
lar1_raw: Final = request_kwargs.get("metadata", {}).get("lar1", {})
|
||||
if not isinstance(lar1_raw, dict):
|
||||
verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata type: %s. Using defaults", type(lar1_raw).__name__)
|
||||
return LAR1Metadata()
|
||||
try:
|
||||
return LAR1Metadata.model_validate(lar1_raw)
|
||||
except ValidationError as exc:
|
||||
verbose_router_logger.warning("[LAR-1] Invalid lar1 metadata: %s. Using defaults", exc)
|
||||
return LAR1Metadata()
|
||||
|
||||
|
||||
class LAR1RoutingStrategy(CustomRoutingStrategyBase):
|
||||
def __init__(
|
||||
self,
|
||||
router_instance: Router | None = None,
|
||||
thresholds: dict[str, float] | None = None,
|
||||
):
|
||||
self._router = router_instance
|
||||
self.thresholds = _normalize_thresholds(thresholds)
|
||||
|
||||
async def async_get_available_deployment(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, str]] | None = None,
|
||||
input: str | list | None = None,
|
||||
specific_deployment: bool | None = False,
|
||||
request_kwargs: dict | None = None,
|
||||
):
|
||||
if request_kwargs is None:
|
||||
request_kwargs = {}
|
||||
if self._router is None:
|
||||
return None
|
||||
|
||||
lar1: Final = _parse_lar1_metadata(request_kwargs)
|
||||
confidence: Final = lar1.confidence
|
||||
evidence: Final = tuple(e.value for e in lar1.evidence)
|
||||
time_dim: Final = lar1.time.value
|
||||
|
||||
healthy: Final = await self._router.async_get_healthy_deployments(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
if isinstance(healthy, dict):
|
||||
return healthy
|
||||
|
||||
if not healthy:
|
||||
return None
|
||||
|
||||
target: Final = self._classify_request(confidence, evidence, time_dim)
|
||||
selected, exact_match = self._select_deployment(target, healthy)
|
||||
|
||||
if selected is None:
|
||||
return None
|
||||
if exact_match:
|
||||
verbose_router_logger.info("[LAR-1] confidence=%s -> %s", confidence, target)
|
||||
else:
|
||||
actual_type: Final = selected.get("model_info", {}).get("type", "unknown")
|
||||
verbose_router_logger.warning(
|
||||
"[LAR-1] No deployment for type '%s', fallback to deployment type '%s'", target, actual_type
|
||||
)
|
||||
return selected
|
||||
|
||||
def _classify_request(
|
||||
self,
|
||||
confidence: float,
|
||||
evidence: tuple[str, ...],
|
||||
time_dim: str,
|
||||
) -> str:
|
||||
if "UNVERIFIED" in evidence:
|
||||
return "cloud-smart"
|
||||
|
||||
if time_dim == LAR1Time.MEM.value:
|
||||
return "cloud-fast"
|
||||
|
||||
t: Final = self.thresholds
|
||||
if confidence < t["low"]:
|
||||
return "cloud-smart"
|
||||
if confidence < t["medium"]:
|
||||
return "cloud-fast"
|
||||
if confidence < t["high"]:
|
||||
return "local"
|
||||
return "deep"
|
||||
|
||||
def _select_deployment(
|
||||
self,
|
||||
target_type: str,
|
||||
deployments: list[dict],
|
||||
) -> tuple[dict | None, bool]:
|
||||
if not deployments:
|
||||
return None, False
|
||||
|
||||
for deployment in deployments:
|
||||
if not isinstance(deployment, dict):
|
||||
continue
|
||||
model_type = deployment.get("model_info", {}).get("type", "")
|
||||
if model_type == target_type:
|
||||
return deployment, True
|
||||
|
||||
for deployment in deployments:
|
||||
if isinstance(deployment, dict):
|
||||
return deployment, False
|
||||
|
||||
return None, False
|
||||
|
||||
def get_available_deployment(self, *args, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"LAR-1 routing only supports async routing. Enable async_only_mode on the router or use acompletion."
|
||||
)
|
||||
|
|
@ -17,7 +17,7 @@ def apply_routing_group_priority(
|
|||
return prioritized
|
||||
|
||||
|
||||
VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy))
|
||||
VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", *(s.value for s in RoutingStrategy))
|
||||
|
||||
|
||||
def validate_routing_strategy(routing_strategy: RoutingStrategy | str | None) -> None:
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
from enum import Enum
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
||||
class LAR1Act(str, Enum):
|
||||
INF = "INF"
|
||||
OBS = "OBS"
|
||||
RET = "RET"
|
||||
GEN = "GEN"
|
||||
|
||||
|
||||
class LAR1Time(str, Enum):
|
||||
NOW = "NOW"
|
||||
MEM = "MEM"
|
||||
CTX = "CTX"
|
||||
PRE = "PRE"
|
||||
|
||||
|
||||
class LAR1Mind(str, Enum):
|
||||
REF = "REF"
|
||||
REC = "REC"
|
||||
HYP = "HYP"
|
||||
ACT = "ACT"
|
||||
|
||||
|
||||
class LAR1Evidence(str, Enum):
|
||||
SYNTH = "SYNTH"
|
||||
RETRIEVED = "RETRIEVED"
|
||||
UNVERIFIED = "UNVERIFIED"
|
||||
CONFIRMED = "CONFIRMED"
|
||||
|
||||
|
||||
class LAR1Metadata(LiteLLMBaseModel):
|
||||
act: LAR1Act = LAR1Act.INF
|
||||
time: LAR1Time = LAR1Time.NOW
|
||||
mind: LAR1Mind = LAR1Mind.REF
|
||||
confidence: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
evidence: list[LAR1Evidence] = Field(default=[])
|
||||
|
|
@ -49,7 +49,6 @@ RoutingStrategyName: TypeAlias = Literal[
|
|||
"latency-based-routing",
|
||||
"cost-based-routing",
|
||||
"usage-based-routing-v2",
|
||||
"lar1",
|
||||
]
|
||||
|
||||
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
|
||||
|
|
|
|||
|
|
@ -1,460 +0,0 @@
|
|||
import pytest
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm import Router
|
||||
from litellm.router_strategy.lar1_routing import (
|
||||
LAR1RoutingStrategy,
|
||||
_normalize_thresholds,
|
||||
_parse_lar1_metadata,
|
||||
apply_lar1_routing_strategy,
|
||||
lar1_thresholds_from_args,
|
||||
DEFAULT_THRESHOLDS,
|
||||
)
|
||||
|
||||
|
||||
def _model_list():
|
||||
return [
|
||||
{
|
||||
"model_name": "agent-router",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {"id": "cloud-smart", "type": "cloud-smart"},
|
||||
},
|
||||
{
|
||||
"model_name": "agent-router",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {"id": "cloud-fast", "type": "cloud-fast"},
|
||||
},
|
||||
{
|
||||
"model_name": "agent-router",
|
||||
"litellm_params": {
|
||||
"model": "ollama/qwythos",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {"id": "local", "type": "local"},
|
||||
},
|
||||
{
|
||||
"model_name": "agent-router",
|
||||
"litellm_params": {
|
||||
"model": "ollama/mythos",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {"id": "deep", "type": "deep"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _create_test_router():
|
||||
return Router(model_list=_model_list())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_low_confidence_routes_to_cloud_smart():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.2}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "cloud-smart"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_high_confidence_routes_to_deep():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.8}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "deep"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unverified_evidence_fallback():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={
|
||||
"metadata": {
|
||||
"lar1": {"confidence": 0.9, "evidence": ["UNVERIFIED"]},
|
||||
}
|
||||
},
|
||||
)
|
||||
assert result["model_info"]["type"] == "cloud-smart"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_thresholds():
|
||||
router = _create_test_router()
|
||||
custom_strategy = LAR1RoutingStrategy(
|
||||
router,
|
||||
thresholds={"low": 0.1, "medium": 0.3, "high": 0.9},
|
||||
)
|
||||
|
||||
result = await custom_strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.85}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mem_time_routes_to_cloud_fast():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={
|
||||
"metadata": {
|
||||
"lar1": {"confidence": 0.9, "time": "MEM"},
|
||||
}
|
||||
},
|
||||
)
|
||||
assert result["model_info"]["type"] == "cloud-fast"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_confidence_falls_back_to_local():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={
|
||||
"metadata": {"lar1": {"confidence": "not-a-number"}},
|
||||
},
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_metadata_defaults_to_local():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={},
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_init_with_lar1_routing_strategy():
|
||||
router = Router(
|
||||
model_list=_model_list(),
|
||||
routing_strategy="lar1",
|
||||
routing_strategy_args={
|
||||
"confidence_threshold_low": 0.1,
|
||||
"confidence_threshold_medium": 0.3,
|
||||
"confidence_threshold_high": 0.9,
|
||||
},
|
||||
)
|
||||
|
||||
result = await router.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.85}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
assert router.routing_strategy == "lar1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_init_lar1_default_thresholds():
|
||||
router = Router(model_list=_model_list(), routing_strategy="lar1")
|
||||
|
||||
result = await router.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.4}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "cloud-fast"
|
||||
assert router.routing_strategy == "lar1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mid_confidence_routes_to_cloud_fast():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.4}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "cloud-fast"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_router_returns_none():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
|
||||
result = await strategy.async_get_available_deployment(model="agent-router")
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_kwargs_none_uses_defaults():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs=None,
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_lar1_metadata_type_uses_defaults(caplog):
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": "not-a-dict"}},
|
||||
)
|
||||
|
||||
assert result["model_info"]["type"] == "local"
|
||||
assert "Invalid lar1 metadata type" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_confirmed_evidence_routes_by_confidence():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={
|
||||
"metadata": {
|
||||
"lar1": {"confidence": 0.8, "evidence": ["CONFIRMED"]},
|
||||
}
|
||||
},
|
||||
)
|
||||
assert result["model_info"]["type"] == "deep"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_healthy_deployments_returns_none():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
router.async_get_healthy_deployments = AsyncMock(return_value=[])
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.8}}},
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_specific_deployment_dict_short_circuit():
|
||||
deployment = {
|
||||
"model_info": {"type": "deep"},
|
||||
"litellm_params": {"model": "ollama/mythos"},
|
||||
}
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
router.async_get_healthy_deployments = AsyncMock(return_value=deployment)
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
specific_deployment=True,
|
||||
request_kwargs={},
|
||||
)
|
||||
assert result == deployment
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_dict_healthy_deployment_returns_none():
|
||||
router = _create_test_router()
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
router.async_get_healthy_deployments = AsyncMock(return_value=[None])
|
||||
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={},
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_parse_lar1_metadata_defaults_when_missing():
|
||||
metadata = _parse_lar1_metadata({})
|
||||
assert metadata.confidence == 0.5
|
||||
assert metadata.time.value == "NOW"
|
||||
|
||||
|
||||
def test_select_deployment_empty_list():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
selected, exact_match = strategy._select_deployment("local", [])
|
||||
assert selected is None
|
||||
assert exact_match is False
|
||||
|
||||
|
||||
def test_select_deployment_skips_non_dict_entries():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
deployment = {"model_info": {"type": "local"}}
|
||||
selected, exact_match = strategy._select_deployment(
|
||||
"local",
|
||||
["skip-me", deployment],
|
||||
)
|
||||
assert selected == deployment
|
||||
assert exact_match is True
|
||||
|
||||
|
||||
def test_select_deployment_fallback_uses_first_dict():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
deployment = {"model_info": {"type": "local"}}
|
||||
selected, exact_match = strategy._select_deployment(
|
||||
"cloud-smart",
|
||||
["skip-me", deployment],
|
||||
)
|
||||
assert selected == deployment
|
||||
assert exact_match is False
|
||||
|
||||
|
||||
def test_select_deployment_all_non_dict_returns_none():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
selected, exact_match = strategy._select_deployment("local", ["a", None])
|
||||
assert selected is None
|
||||
assert exact_match is False
|
||||
|
||||
|
||||
def test_lar1_thresholds_from_args_uses_defaults():
|
||||
assert lar1_thresholds_from_args({}) == DEFAULT_THRESHOLDS
|
||||
|
||||
|
||||
def test_lar1_thresholds_from_args_ignores_invalid_values():
|
||||
assert lar1_thresholds_from_args(
|
||||
{
|
||||
"confidence_threshold_low": "bad",
|
||||
"confidence_threshold_medium": 0.4,
|
||||
"confidence_threshold_high": 0.8,
|
||||
}
|
||||
) == {"low": 0.3, "medium": 0.4, "high": 0.8}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_settings_switches_to_lar1_routing():
|
||||
router = Router(model_list=_model_list(), routing_strategy="simple-shuffle")
|
||||
router.update_settings(
|
||||
routing_strategy="lar1",
|
||||
routing_strategy_args={
|
||||
"confidence_threshold_low": 0.1,
|
||||
"confidence_threshold_medium": 0.3,
|
||||
"confidence_threshold_high": 0.9,
|
||||
},
|
||||
)
|
||||
|
||||
result = await router.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.85}}},
|
||||
)
|
||||
assert router.routing_strategy == "lar1"
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
def test_update_settings_switching_from_lar1_restores_default_selectors():
|
||||
router = Router(model_list=_model_list(), routing_strategy="lar1")
|
||||
|
||||
router.update_settings(routing_strategy="simple-shuffle")
|
||||
|
||||
assert router.routing_strategy == "simple-shuffle"
|
||||
assert "get_available_deployment" not in router.__dict__
|
||||
assert "async_get_available_deployment" not in router.__dict__
|
||||
result = router.get_available_deployment(model="agent-router")
|
||||
assert result["model_name"] == "agent-router"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_settings_routing_strategy_args_relinks_lar1():
|
||||
router = Router(model_list=_model_list(), routing_strategy="lar1")
|
||||
|
||||
router.update_settings(
|
||||
routing_strategy_args={
|
||||
"confidence_threshold_low": 0.1,
|
||||
"confidence_threshold_medium": 0.3,
|
||||
"confidence_threshold_high": 0.9,
|
||||
},
|
||||
)
|
||||
|
||||
result = await router.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.85}}},
|
||||
)
|
||||
assert result["model_info"]["type"] == "local"
|
||||
|
||||
|
||||
def test_apply_lar1_routing_strategy_wires_custom_selector():
|
||||
router = Router(model_list=_model_list(), routing_strategy="simple-shuffle")
|
||||
apply_lar1_routing_strategy(router, {"confidence_threshold_high": 0.9})
|
||||
assert router.routing_strategy == "lar1"
|
||||
with pytest.raises(NotImplementedError, match="async routing"):
|
||||
router.get_available_deployment(model="agent-router")
|
||||
|
||||
|
||||
def test_apply_lar1_invalid_thresholds_leaves_router_unchanged():
|
||||
router = Router(model_list=_model_list(), routing_strategy="simple-shuffle")
|
||||
|
||||
with pytest.raises(ValueError, match="LAR-1 thresholds must satisfy"):
|
||||
apply_lar1_routing_strategy(
|
||||
router,
|
||||
{
|
||||
"confidence_threshold_low": 0.9,
|
||||
"confidence_threshold_medium": 0.5,
|
||||
"confidence_threshold_high": 0.7,
|
||||
},
|
||||
)
|
||||
|
||||
assert router.routing_strategy == "simple-shuffle"
|
||||
assert "async_get_available_deployment" not in router.__dict__
|
||||
result = router.get_available_deployment(model="agent-router")
|
||||
assert result["model_name"] == "agent-router"
|
||||
|
||||
|
||||
def test_invalid_threshold_order_raises():
|
||||
with pytest.raises(ValueError, match="LAR-1 thresholds must satisfy"):
|
||||
_normalize_thresholds({"low": 0.5, "medium": 0.3, "high": 0.7})
|
||||
|
||||
|
||||
def test_get_available_deployment_raises_not_implemented():
|
||||
strategy = LAR1RoutingStrategy()
|
||||
with pytest.raises(NotImplementedError, match="async routing"):
|
||||
strategy.get_available_deployment(model="agent-router")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_target_type_falls_back_with_warning(caplog):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "agent-router",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "fake-key",
|
||||
},
|
||||
"model_info": {"id": "only-local", "type": "local"},
|
||||
}
|
||||
]
|
||||
)
|
||||
strategy = LAR1RoutingStrategy(router)
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
result = await strategy.async_get_available_deployment(
|
||||
model="agent-router",
|
||||
request_kwargs={"metadata": {"lar1": {"confidence": 0.2}}},
|
||||
)
|
||||
|
||||
assert result["model_info"]["type"] == "local"
|
||||
assert "No deployment for type 'cloud-smart'" in caplog.text
|
||||
|
|
@ -781,7 +781,7 @@ def test_request_routing_strategy_override_helper_validates_directly():
|
|||
router = _build_router(routing_strategy="least-busy")
|
||||
assert router._get_request_routing_strategy_override({"routing_strategy": "simple-shuffle"}) == "simple-shuffle"
|
||||
assert router._get_request_routing_strategy_override({"routing_strategy": RoutingStrategy.LEAST_BUSY}) == "least-busy"
|
||||
assert router._get_request_routing_strategy_override({"routing_strategy": "lar1"}) is None
|
||||
assert router._get_request_routing_strategy_override({"routing_strategy": "unknown-strategy"}) is None
|
||||
assert router._get_request_routing_strategy_override({"routing_strategy": {"bad": "type"}}) is None
|
||||
assert router._get_request_routing_strategy_override({}) is None
|
||||
assert router._get_request_routing_strategy_override(None) is None
|
||||
|
|
|
|||
|
|
@ -2444,23 +2444,12 @@ def test_get_model_list_of_routed_group_falls_back_to_the_wildcard_route_serving
|
|||
assert _served_models(router.get_model_list(model_name="openai/gpt-4o")) == ["ollama/qwen3:0.6b"]
|
||||
|
||||
|
||||
def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selector():
|
||||
def test_switch_routing_strategy_installs_usage_based_v2_selector():
|
||||
router = _alias_cost_router()
|
||||
|
||||
router._switch_routing_strategy(
|
||||
"lar1",
|
||||
{
|
||||
"routing_strategy_args": {
|
||||
"confidence_threshold_low": 0.1,
|
||||
"confidence_threshold_medium": 0.3,
|
||||
"confidence_threshold_high": 0.9,
|
||||
}
|
||||
},
|
||||
)
|
||||
assert router.routing_strategy == "lar1"
|
||||
assert "async_get_available_deployment" in router.__dict__
|
||||
router.update_settings(routing_strategy="usage-based-routing-v2")
|
||||
|
||||
router._switch_routing_strategy("usage-based-routing-v2", {})
|
||||
assert router.routing_strategy == "usage-based-routing-v2"
|
||||
assert router.lowesttpm_logger_v2 is not None
|
||||
assert "async_get_available_deployment" not in router.__dict__
|
||||
|
||||
|
|
|
|||
|
|
@ -637,7 +637,6 @@ def test_owned_leaf_literals_reject_unknown_values(leaf: type, sample: Mapping[s
|
|||
"latency-based-routing",
|
||||
"cost-based-routing",
|
||||
"usage-based-routing-v2",
|
||||
"lar1",
|
||||
],
|
||||
)
|
||||
def test_routing_options_accept_every_strategy_the_router_accepts(strategy: str) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue