mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
45fa78470d
The text-completion wire test set litellm.aclient_session, which the test-quality gate (TQ005) flags as a process-wide global write. Pass an AsyncOpenAI client through the router's client kwarg instead, so the test owns its transport and needs no cache flush or global restore. Claude-Session: https://claude.ai/code/session_01XKkTFa6g7Rmd6vtHL91GMn
660 lines
22 KiB
Python
660 lines
22 KiB
Python
"""
|
|
Tests for order-based fallback routing.
|
|
|
|
When deployments have `order` set in litellm_params, lower order deployments
|
|
should be tried first, and higher order deployments should be used as fallbacks
|
|
when lower order deployments fail.
|
|
"""
|
|
|
|
import json
|
|
from typing import Final, Optional
|
|
|
|
import httpx
|
|
import pytest
|
|
from openai import AsyncOpenAI
|
|
|
|
import litellm
|
|
from litellm import Router
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
from litellm.router_utils.prompt_caching_cache import PromptCachingCache
|
|
from litellm.types.router import RouterRateLimitError
|
|
from litellm.utils import _get_deployment_order, _get_order_filtered_deployments
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unit tests for _get_order_filtered_deployments
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestGetOrderFilteredDeployments:
|
|
def _make_deployment(self, order: Optional[int], dep_id: str) -> dict:
|
|
params: dict = {"model": "gpt-4o", "api_key": "key"}
|
|
if order is not None:
|
|
params["order"] = order
|
|
return {
|
|
"model_name": "test-model",
|
|
"litellm_params": params,
|
|
"model_info": {"id": dep_id},
|
|
}
|
|
|
|
def test_returns_min_order_group(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
self._make_deployment(1, "c"),
|
|
]
|
|
result = _get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
assert all(d["model_info"]["id"] in ("a", "c") for d in result)
|
|
|
|
def test_target_order_filters_to_exact_level(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
self._make_deployment(3, "c"),
|
|
]
|
|
result = _get_order_filtered_deployments(deps, target_order=2)
|
|
assert len(result) == 1
|
|
assert result[0]["model_info"]["id"] == "b"
|
|
|
|
def test_target_order_no_match_returns_empty(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
]
|
|
result = _get_order_filtered_deployments(deps, target_order=99)
|
|
assert result == []
|
|
|
|
def test_target_order_no_match_does_not_reselect_lower_order(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
]
|
|
remaining_after_pre_call = [deps[0]]
|
|
result = _get_order_filtered_deployments(remaining_after_pre_call, target_order=2)
|
|
assert result == []
|
|
|
|
def test_no_order_set_returns_all(self):
|
|
deps = [
|
|
self._make_deployment(None, "a"),
|
|
self._make_deployment(None, "b"),
|
|
]
|
|
result = _get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
|
|
def test_empty_list(self):
|
|
result = _get_order_filtered_deployments([])
|
|
assert result == []
|
|
|
|
def test_single_order_returns_all_with_that_order(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(1, "b"),
|
|
]
|
|
result = _get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Integration tests for order-based fallback in Router
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_router_order_without_pre_call_checks():
|
|
"""Order filtering should work even when enable_pre_call_checks=False (default)."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 1",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
enable_pre_call_checks=False,
|
|
)
|
|
|
|
for _ in range(20):
|
|
response = router.completion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "1"
|
|
|
|
|
|
def test_router_order_no_fallback_when_healthy():
|
|
"""When order=1 is healthy, order=2 should never be used."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 1",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
for _ in range(50):
|
|
response = router.completion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "1"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_on_failure():
|
|
"""When order=1 fails, order=2 should be tried as fallback."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad-key",
|
|
"mock_response": Exception("connection error"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good-key",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_three_levels():
|
|
"""When order=1 and order=2 both fail, order=3 should be tried."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 3",
|
|
"order": 3,
|
|
},
|
|
"model_info": {"id": "3"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_then_external_fallback():
|
|
"""When all order levels fail, external fallbacks should be tried."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "fallback-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from external fallback",
|
|
},
|
|
"model_info": {"id": "fallback"},
|
|
},
|
|
],
|
|
fallbacks=[{"test-model": ["fallback-model"]}],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "fallback"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_non_standard_fallbacks():
|
|
"""Non-standard fallback formats (e.g. fallbacks=["model-name"]) passed
|
|
per-request should still be tried after all order levels are exhausted."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "fallback-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from non-standard fallback",
|
|
},
|
|
"model_info": {"id": "fallback"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
fallbacks=["fallback-model"], # non-standard format, passed per-request
|
|
)
|
|
assert response._hidden_params["model_id"] == "fallback"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_wildcard_model_group():
|
|
"""Wildcard model groups should also advance across order levels."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "openai/*",
|
|
"litellm_params": {
|
|
"model": "openai/*",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "openai/*",
|
|
"litellm_params": {
|
|
"model": "openai/*",
|
|
"api_key": "good",
|
|
"mock_response": "success from wildcard order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="openai/gpt-4.1-mini",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_hidden_model_group_alias():
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "canonical-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "canonical-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
model_group_alias={"hidden-alias": {"model": "canonical-model", "hidden": True}},
|
|
num_retries=0,
|
|
)
|
|
|
|
assert "hidden-alias" not in {deployment["model_name"] for deployment in router.get_model_list() or []}
|
|
|
|
response = await router.acompletion(
|
|
model="hidden-alias",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_does_not_reselect_order_1_when_order_2_is_filtered_out():
|
|
class _DropOrder2(CustomLogger):
|
|
async def async_filter_deployments(
|
|
self, model, healthy_deployments, messages, request_kwargs=None, parent_otel_span=None
|
|
):
|
|
return [d for d in healthy_deployments if _get_deployment_order(d) != 2]
|
|
|
|
drop_order_2: Final = _DropOrder2()
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "litellm.RateLimitError",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
litellm.callbacks.append(drop_order_2)
|
|
try:
|
|
with pytest.raises(RouterRateLimitError, match="No deployments available") as exc_info:
|
|
await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert "success from order 2" not in str(exc_info.value)
|
|
finally:
|
|
litellm.callbacks.remove(drop_order_2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_ignores_prompt_cache_pin_on_target_order():
|
|
messages = [{"role": "user", "content": "word " * 5000}]
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("azure peak load"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
optional_pre_call_checks=["prompt_caching"],
|
|
)
|
|
await PromptCachingCache(cache=router.cache).async_add_model_id(
|
|
model_id="1",
|
|
messages=messages,
|
|
tools=None,
|
|
)
|
|
response = await router.acompletion(model="test-model", messages=messages)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_retries_keep_target_order():
|
|
seen_target_orders: Final = []
|
|
|
|
class _RecordTargetOrder(CustomLogger):
|
|
async def async_filter_deployments(
|
|
self, model, healthy_deployments, messages, request_kwargs=None, parent_otel_span=None
|
|
):
|
|
seen_target_orders.append((request_kwargs or {}).get("_target_order"))
|
|
return healthy_deployments
|
|
|
|
recorder: Final = _RecordTargetOrder()
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=1,
|
|
)
|
|
litellm.callbacks.append(recorder)
|
|
try:
|
|
with pytest.raises(Exception, match="fail order 2"):
|
|
await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
finally:
|
|
litellm.callbacks.remove(recorder)
|
|
assert seen_target_orders.count(2) >= 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generic_api_call_strips_target_order_from_provider_kwargs():
|
|
captured: Final = {}
|
|
|
|
async def _fake_provider(**provider_kwargs):
|
|
captured.update(provider_kwargs)
|
|
return "ok"
|
|
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
)
|
|
response = await router._ageneric_api_call_with_fallbacks_helper(
|
|
model="test-model",
|
|
original_generic_function=_fake_provider,
|
|
_target_order=2,
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response == "ok"
|
|
assert captured["model"] == "gpt-4o"
|
|
assert "_target_order" not in captured
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_text_completion_order_fallback_hop_does_not_send_target_order_upstream():
|
|
upstream_bodies: Final[list[dict]] = []
|
|
|
|
def _upstream(request: httpx.Request) -> httpx.Response:
|
|
upstream_bodies.append(json.loads(request.content))
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "cmpl-1",
|
|
"object": "text_completion",
|
|
"created": 0,
|
|
"model": "gpt-3.5-turbo-instruct",
|
|
"choices": [{"text": "ok from order 2", "index": 0, "logprobs": None, "finish_reason": "stop"}],
|
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
|
},
|
|
)
|
|
|
|
upstream_client: Final = AsyncOpenAI(
|
|
api_key="key",
|
|
base_url="http://upstream.test",
|
|
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_upstream)),
|
|
)
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
|
"api_key": "key",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
|
"api_key": "key",
|
|
"api_base": "http://upstream.test",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
try:
|
|
response = await router.atext_completion(model="test-model", prompt="hi", client=upstream_client)
|
|
finally:
|
|
await upstream_client.close()
|
|
|
|
assert response._hidden_params["model_id"] == "2"
|
|
assert upstream_bodies
|
|
assert all("_target_order" not in body for body in upstream_bodies)
|
|
|
|
|
|
def test_check_non_standard_fallback_format():
|
|
from litellm.router_utils.fallback_event_handlers import (
|
|
_check_non_standard_fallback_format,
|
|
)
|
|
|
|
# Standard formats
|
|
assert _check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == False
|
|
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"]}]) == False
|
|
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "region": ["us-east-1"]}]) == False
|
|
|
|
# Non-standard formats
|
|
assert _check_non_standard_fallback_format([{"model": "qwen-backup"}]) == True
|
|
assert (
|
|
_check_non_standard_fallback_format([{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}])
|
|
== True
|
|
)
|
|
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "api_key": "some-key"}]) == True
|