mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
8a176c0f0a
* fix(router): drop the encrypted reasoning a fallback hop's target cannot decrypt An order-based or configured fallback hop replayed the failed provider's encrypted reasoning items to the next deployment, which answered 400 (Bedrock Mantle: invalid encrypted reasoning; OpenAI: invalid_encrypted_content), so every multi-turn Responses fallback for Codex-style clients failed. The hop now drops the encrypted reasoning its target cannot decrypt and keeps each item's readable summary. With encrypted_content_affinity on, the pin narrows to the hop's target order instead of emptying it, so the hop reaches the next order instead of failing with no deployments available. * fix(router): keep encrypted reasoning a same-boundary fallback hop can decrypt Unmarked encrypted reasoning on a hop is attributed to the deployment that just failed, read from the retry breadcrumb, so a hop to a deployment on the same api_base and api_key keeps it and a cross-provider hop still drops it. The hop tests script the upstream at the httpx boundary instead of doubling the handler, and the router coverage script lists the two hop helpers with their tests * fix(router): read the hop's failed deployment from its own metadata bucket and carry it into the Responses mid-stream snapshot * test(router): use a real Router without the origin deployment in the hop strip test * test(router): cover the hop strip when no failed deployment is known * fix(proxy): drop the router's fallback hop state keys from the client body * fix(proxy): keep a request's max_fallbacks cap, drop only the hop state keys * test(integration): audit cells for the fallback hop encrypted reasoning strip Forty-five checked-in cells under tests/integration/routing prove the hop strips the previous deployment's encrypted reasoning on /v1/responses, /v1/chat/completions and /v1/messages (httpx, OpenAI and Anthropic SDKs, sync and async, streaming and not), that the affinity pin yields to the hop, that a client-sent fallback_depth, _target_order and attempted_targets never move or strip a request, and that a concurrent burst, an order-1 outage and a killed worker keep every request stripped and logged once. Every call goes through a lane pinned to one worker that already lists the deployments it needs, because the peer worker learns a /model/new row through the config-sync resync up to sixteen seconds later --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
830 lines
28 KiB
Python
830 lines
28 KiB
Python
"""
|
|
Tests for order-based fallback routing.
|
|
|
|
When deployments have `order` set in litellm_params, lower order deployments
|
|
should be tried first, and higher order deployments should be used as fallbacks
|
|
when lower order deployments fail.
|
|
"""
|
|
|
|
import json
|
|
from typing import Final, Optional
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
from openai import AsyncOpenAI
|
|
|
|
import litellm
|
|
from litellm import Router
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
from litellm.router_utils.prompt_caching_cache import PromptCachingCache
|
|
from litellm.types.router import RouterRateLimitError
|
|
from litellm.utils import get_deployment_order, get_order_filtered_deployments
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unit tests for get_order_filtered_deployments
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestGetOrderFilteredDeployments:
|
|
def _make_deployment(self, order: Optional[int], dep_id: str) -> dict:
|
|
params: dict = {"model": "gpt-4o", "api_key": "key"}
|
|
if order is not None:
|
|
params["order"] = order
|
|
return {
|
|
"model_name": "test-model",
|
|
"litellm_params": params,
|
|
"model_info": {"id": dep_id},
|
|
}
|
|
|
|
def test_returns_min_order_group(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
self._make_deployment(1, "c"),
|
|
]
|
|
result = get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
assert all(d["model_info"]["id"] in ("a", "c") for d in result)
|
|
|
|
def test_target_order_filters_to_exact_level(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
self._make_deployment(3, "c"),
|
|
]
|
|
result = get_order_filtered_deployments(deps, target_order=2)
|
|
assert len(result) == 1
|
|
assert result[0]["model_info"]["id"] == "b"
|
|
|
|
def test_target_order_no_match_returns_empty(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
]
|
|
result = get_order_filtered_deployments(deps, target_order=99)
|
|
assert result == []
|
|
|
|
def test_target_order_no_match_does_not_reselect_lower_order(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(2, "b"),
|
|
]
|
|
remaining_after_pre_call = [deps[0]]
|
|
result = get_order_filtered_deployments(remaining_after_pre_call, target_order=2)
|
|
assert result == []
|
|
|
|
def test_no_order_set_returns_all(self):
|
|
deps = [
|
|
self._make_deployment(None, "a"),
|
|
self._make_deployment(None, "b"),
|
|
]
|
|
result = get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
|
|
def test_empty_list(self):
|
|
result = get_order_filtered_deployments([])
|
|
assert result == []
|
|
|
|
def test_single_order_returns_all_with_that_order(self):
|
|
deps = [
|
|
self._make_deployment(1, "a"),
|
|
self._make_deployment(1, "b"),
|
|
]
|
|
result = get_order_filtered_deployments(deps)
|
|
assert len(result) == 2
|
|
|
|
|
|
def test_get_deployment_order_returns_unvalidated_order():
|
|
assert get_deployment_order({"litellm_params": {"order": "first"}}) == "first"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Integration tests for order-based fallback in Router
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_router_order_without_pre_call_checks():
|
|
"""Order filtering should work even when enable_pre_call_checks=False (default)."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 1",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
enable_pre_call_checks=False,
|
|
)
|
|
|
|
for _ in range(20):
|
|
response = router.completion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "1"
|
|
|
|
|
|
def test_router_order_no_fallback_when_healthy():
|
|
"""When order=1 is healthy, order=2 should never be used."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 1",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
for _ in range(50):
|
|
response = router.completion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "1"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_on_failure():
|
|
"""When order=1 fails, order=2 should be tried as fallback."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad-key",
|
|
"mock_response": Exception("connection error"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good-key",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_three_levels():
|
|
"""When order=1 and order=2 both fail, order=3 should be tried."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 3",
|
|
"order": 3,
|
|
},
|
|
"model_info": {"id": "3"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "3"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_then_external_fallback():
|
|
"""When all order levels fail, external fallbacks should be tried."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "fallback-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from external fallback",
|
|
},
|
|
"model_info": {"id": "fallback"},
|
|
},
|
|
],
|
|
fallbacks=[{"test-model": ["fallback-model"]}],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "fallback"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_non_standard_fallbacks():
|
|
"""Non-standard fallback formats (e.g. fallbacks=["model-name"]) passed
|
|
per-request should still be tried after all order levels are exhausted."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "fallback-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from non-standard fallback",
|
|
},
|
|
"model_info": {"id": "fallback"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
fallbacks=["fallback-model"], # non-standard format, passed per-request
|
|
)
|
|
assert response._hidden_params["model_id"] == "fallback"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_wildcard_model_group():
|
|
"""Wildcard model groups should also advance across order levels."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "openai/*",
|
|
"litellm_params": {
|
|
"model": "openai/*",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "openai/*",
|
|
"litellm_params": {
|
|
"model": "openai/*",
|
|
"api_key": "good",
|
|
"mock_response": "success from wildcard order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
response = await router.acompletion(
|
|
model="openai/gpt-4.1-mini",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_with_hidden_model_group_alias():
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "canonical-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "canonical-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
model_group_alias={"hidden-alias": {"model": "canonical-model", "hidden": True}},
|
|
num_retries=0,
|
|
)
|
|
|
|
assert "hidden-alias" not in {deployment["model_name"] for deployment in router.get_model_list() or []}
|
|
|
|
response = await router.acompletion(
|
|
model="hidden-alias",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_does_not_reselect_order_1_when_order_2_is_filtered_out():
|
|
class _DropOrder2(CustomLogger):
|
|
async def async_filter_deployments(
|
|
self, model, healthy_deployments, messages, request_kwargs=None, parent_otel_span=None
|
|
):
|
|
return [d for d in healthy_deployments if get_deployment_order(d) != 2]
|
|
|
|
drop_order_2: Final = _DropOrder2()
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "litellm.RateLimitError",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "key",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
litellm.callbacks.append(drop_order_2)
|
|
try:
|
|
with pytest.raises(RouterRateLimitError, match="No deployments available") as exc_info:
|
|
await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert "success from order 2" not in str(exc_info.value)
|
|
finally:
|
|
litellm.callbacks.remove(drop_order_2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_ignores_prompt_cache_pin_on_target_order():
|
|
messages = [{"role": "user", "content": "word " * 5000}]
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("azure peak load"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "good",
|
|
"mock_response": "success from order 2",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
optional_pre_call_checks=["prompt_caching"],
|
|
)
|
|
await PromptCachingCache(cache=router.cache).async_add_model_id(
|
|
model_id="1",
|
|
messages=messages,
|
|
tools=None,
|
|
)
|
|
response = await router.acompletion(model="test-model", messages=messages)
|
|
assert response._hidden_params["model_id"] == "2"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_router_order_fallback_retries_keep_target_order():
|
|
seen_target_orders: Final = []
|
|
|
|
class _RecordTargetOrder(CustomLogger):
|
|
async def async_filter_deployments(
|
|
self, model, healthy_deployments, messages, request_kwargs=None, parent_otel_span=None
|
|
):
|
|
seen_target_orders.append((request_kwargs or {}).get("_target_order"))
|
|
return healthy_deployments
|
|
|
|
recorder: Final = _RecordTargetOrder()
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o",
|
|
"api_key": "bad",
|
|
"mock_response": Exception("fail order 2"),
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=1,
|
|
)
|
|
litellm.callbacks.append(recorder)
|
|
try:
|
|
with pytest.raises(Exception, match="fail order 2"):
|
|
await router.acompletion(
|
|
model="test-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
finally:
|
|
litellm.callbacks.remove(recorder)
|
|
assert seen_target_orders.count(2) >= 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generic_api_call_strips_target_order_from_provider_kwargs():
|
|
captured: Final = {}
|
|
|
|
async def _fake_provider(**provider_kwargs):
|
|
captured.update(provider_kwargs)
|
|
return "ok"
|
|
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
)
|
|
response = await router._ageneric_api_call_with_fallbacks_helper(
|
|
model="test-model",
|
|
original_generic_function=_fake_provider,
|
|
_target_order=2,
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
)
|
|
assert response == "ok"
|
|
assert captured["model"] == "gpt-4o"
|
|
assert "_target_order" not in captured
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_text_completion_order_fallback_hop_does_not_send_target_order_upstream():
|
|
upstream_bodies: Final[list[dict]] = []
|
|
|
|
def _upstream(request: httpx.Request) -> httpx.Response:
|
|
upstream_bodies.append(json.loads(request.content))
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "cmpl-1",
|
|
"object": "text_completion",
|
|
"created": 0,
|
|
"model": "gpt-3.5-turbo-instruct",
|
|
"choices": [{"text": "ok from order 2", "index": 0, "logprobs": None, "finish_reason": "stop"}],
|
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
|
},
|
|
)
|
|
|
|
upstream_client: Final = AsyncOpenAI(
|
|
api_key="key",
|
|
base_url="http://upstream.test",
|
|
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_upstream)),
|
|
)
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
|
"api_key": "key",
|
|
"mock_response": Exception("fail order 1"),
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
|
"api_key": "key",
|
|
"api_base": "http://upstream.test",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
try:
|
|
response = await router.atext_completion(model="test-model", prompt="hi", client=upstream_client)
|
|
finally:
|
|
await upstream_client.close()
|
|
|
|
assert response._hidden_params["model_id"] == "2"
|
|
assert upstream_bodies
|
|
assert all("_target_order" not in body for body in upstream_bodies)
|
|
|
|
|
|
|
|
_OPENAI_RESPONSES_URL: Final = "https://api.openai.com/v1/responses"
|
|
_MANTLE_RESPONSES_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses"
|
|
_OVERLOADED_UPSTREAM: Final = {"error": {"message": "overloaded", "type": "server_error", "code": "server_error"}}
|
|
|
|
|
|
def _completed_response_body(response_id: str, model: str, text: str) -> dict[str, object]:
|
|
return {
|
|
"id": response_id,
|
|
"object": "response",
|
|
"created_at": 0,
|
|
"status": "completed",
|
|
"model": model,
|
|
"output": [
|
|
{
|
|
"id": f"msg_{response_id}",
|
|
"type": "message",
|
|
"status": "completed",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
|
}
|
|
],
|
|
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
|
}
|
|
|
|
|
|
def _responses_history_with_order_1_reasoning() -> list[dict]:
|
|
return [
|
|
{"type": "message", "role": "user", "content": "What is 17*23?"},
|
|
{
|
|
"type": "reasoning",
|
|
"id": "rs_order1",
|
|
"encrypted_content": "gAAAAA-minted-by-order-1",
|
|
"summary": [{"type": "summary_text", "text": "multiply 17 by 23"}],
|
|
},
|
|
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "391"}]},
|
|
{"type": "message", "role": "user", "content": "And 19*21?"},
|
|
]
|
|
|
|
|
|
def _responses_history_without_order_1_encrypted_reasoning() -> list[dict]:
|
|
return [
|
|
{"type": "message", "role": "user", "content": "What is 17*23?"},
|
|
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "multiply 17 by 23"}]},
|
|
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "391"}]},
|
|
{"type": "message", "role": "user", "content": "And 19*21?"},
|
|
]
|
|
|
|
|
|
def _openai_then_mantle_order_router() -> Router:
|
|
return Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "gpt-6-astra",
|
|
"litellm_params": {"model": "openai/gpt-6-astra", "api_key": "openai-key", "order": 1},
|
|
"model_info": {"id": "openai-order-1"},
|
|
},
|
|
{
|
|
"model_name": "gpt-6-astra",
|
|
"litellm_params": {
|
|
"model": "bedrock_mantle/openai.gpt-6-astra",
|
|
"api_key": "mantle-bearer-token",
|
|
"aws_region_name": "us-east-1",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "mantle-order-2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
|
|
def _two_openai_orders_on_one_encryption_boundary_router() -> Router:
|
|
return Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "gpt-6-astra",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-6-astra",
|
|
"api_base": "https://api.openai.com/v1",
|
|
"api_key": "openai-key",
|
|
"order": 1,
|
|
},
|
|
"model_info": {"id": "openai-order-1"},
|
|
},
|
|
{
|
|
"model_name": "gpt-6-astra",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-6-astra-mini",
|
|
"api_base": "https://api.openai.com/v1",
|
|
"api_key": "openai-key",
|
|
"order": 2,
|
|
},
|
|
"model_info": {"id": "openai-order-2"},
|
|
},
|
|
],
|
|
num_retries=0,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_responses_order_fallback_hop_drops_the_encrypted_reasoning_the_next_provider_cannot_decrypt(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
|
):
|
|
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
|
openai_route: Final = respx_mock.post(_OPENAI_RESPONSES_URL).mock(
|
|
return_value=httpx.Response(500, json=_OVERLOADED_UPSTREAM)
|
|
)
|
|
mantle_route: Final = respx_mock.post(_MANTLE_RESPONSES_URL).mock(
|
|
return_value=httpx.Response(200, json=_completed_response_body("resp_mantle", "openai.gpt-6-astra", "399"))
|
|
)
|
|
|
|
response = await _openai_then_mantle_order_router().aresponses(
|
|
model="gpt-6-astra", input=_responses_history_with_order_1_reasoning(), store=False
|
|
)
|
|
|
|
assert response._hidden_params["model_id"] == "mantle-order-2"
|
|
assert json.loads(openai_route.calls.last.request.read())["input"] == _responses_history_with_order_1_reasoning()
|
|
assert (
|
|
json.loads(mantle_route.calls.last.request.read())["input"]
|
|
== _responses_history_without_order_1_encrypted_reasoning()
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_responses_order_fallback_hop_keeps_the_encrypted_reasoning_the_same_boundary_can_decrypt(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
|
):
|
|
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
|
openai_route: Final = respx_mock.post(_OPENAI_RESPONSES_URL).mock(
|
|
side_effect=[
|
|
httpx.Response(500, json=_OVERLOADED_UPSTREAM),
|
|
httpx.Response(200, json=_completed_response_body("resp_order2", "gpt-6-astra-mini", "399")),
|
|
]
|
|
)
|
|
|
|
response = await _two_openai_orders_on_one_encryption_boundary_router().aresponses(
|
|
model="gpt-6-astra", input=_responses_history_with_order_1_reasoning(), store=False
|
|
)
|
|
|
|
assert response._hidden_params["model_id"] == "openai-order-2"
|
|
assert [json.loads(call.request.read())["input"] for call in openai_route.calls] == [
|
|
_responses_history_with_order_1_reasoning(),
|
|
_responses_history_with_order_1_reasoning(),
|
|
]
|
|
|
|
|
|
def test_fallback_hop_reads_the_deployment_that_just_failed_from_the_metadata_bucket_it_writes():
|
|
router: Final = _two_openai_orders_on_one_encryption_boundary_router()
|
|
order_2: Final = router.get_deployment(model_id="openai-order-2").model_dump(exclude_none=True)
|
|
hop_input: Final = _responses_history_with_order_1_reasoning()
|
|
hop_kwargs: Final = {
|
|
"model": "gpt-6-astra",
|
|
"input": hop_input,
|
|
"fallback_depth": 1,
|
|
"metadata": {"model_info": {"id": "openai-order-1"}},
|
|
"litellm_metadata": {"previous_models": [{"deployment_id": None}]},
|
|
}
|
|
|
|
router._update_kwargs_with_deployment(deployment=order_2, kwargs=hop_kwargs)
|
|
|
|
assert hop_input == _responses_history_with_order_1_reasoning()
|
|
assert hop_kwargs["metadata"]["model_info"]["id"] == "openai-order-2"
|
|
|
|
|
|
def test_check_non_standard_fallback_format():
|
|
from litellm.router_utils.fallback_event_handlers import (
|
|
check_non_standard_fallback_format,
|
|
)
|
|
|
|
# Standard formats
|
|
assert check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == False
|
|
assert check_non_standard_fallback_format([{"model": ["qwen-backup"]}]) == False
|
|
assert check_non_standard_fallback_format([{"model": ["qwen-backup"], "region": ["us-east-1"]}]) == False
|
|
|
|
# Non-standard formats
|
|
assert check_non_standard_fallback_format([{"model": "qwen-backup"}]) == True
|
|
assert (
|
|
check_non_standard_fallback_format([{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}])
|
|
== True
|
|
)
|
|
assert check_non_standard_fallback_format([{"model": ["qwen-backup"], "api_key": "some-key"}]) == True
|