mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): keep the raw model collision refusal out of router fallbacks
A configured fallback, including a catch-all, could serve a request the check had refused. The refusal is now its own BadRequestError subclass that the fallback handler re-raises. Adds the audit cells: streaming, OpenAI and Anthropic SDK clients, hostile setting values and a worker kill mid burst.
This commit is contained in:
parent
21e41e0044
commit
c5e5bfbab4
4 changed files with 260 additions and 6 deletions
|
|
@ -205,6 +205,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
RawModelGroupCollisionError,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
|
|
@ -8294,6 +8295,7 @@ class Router:
|
|||
or original_model_group is None
|
||||
or is_guardrail_intervention(e)
|
||||
or unsupported_params_without_fallback
|
||||
or isinstance(e, RawModelGroupCollisionError)
|
||||
):
|
||||
raise e
|
||||
|
||||
|
|
|
|||
|
|
@ -339,11 +339,15 @@ def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]:
|
|||
return None, deployment["model_name"]
|
||||
|
||||
|
||||
class RawModelGroupCollisionError(BadRequestError):
|
||||
"""The 400 for a raw `litellm_params.model` that several model groups share; router fallbacks never serve it."""
|
||||
|
||||
|
||||
def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None:
|
||||
"""Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group."""
|
||||
if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2:
|
||||
return
|
||||
raise BadRequestError(
|
||||
raise RawModelGroupCollisionError(
|
||||
message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. "
|
||||
"Send one of the model group names instead",
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Awaitable, Callable, Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value, string_value
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
|
|
@ -27,8 +39,11 @@ TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"})
|
|||
ALIAS: Final = "collision-alias"
|
||||
POOLED_CALLS: Final = 48
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
CATCH_ALL_FALLBACK: Final[dict[str, JsonValue]] = {"fallbacks": [{"*": ["collision-solo"]}]}
|
||||
|
||||
Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]]
|
||||
SdkCall = Callable[[Gateway, str], None]
|
||||
BURST: Final = 30
|
||||
|
||||
|
||||
def _json_reply(body: dict[str, JsonValue]) -> Reply:
|
||||
|
|
@ -114,7 +129,7 @@ def peer() -> Iterator[Wire]:
|
|||
@pytest.fixture(scope="module")
|
||||
def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded")
|
||||
path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True})
|
||||
path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True, **CATCH_ALL_FALLBACK})
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
yield proxy
|
||||
|
||||
|
|
@ -133,7 +148,11 @@ def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.Tem
|
|||
path: Final = _config(
|
||||
peer,
|
||||
directory,
|
||||
{"enable_cross_model_group_collision_check": True, "model_group_alias": {ALIAS: SHARED_MODEL}},
|
||||
{
|
||||
"enable_cross_model_group_collision_check": True,
|
||||
"model_group_alias": {ALIAS: SHARED_MODEL},
|
||||
**CATCH_ALL_FALLBACK,
|
||||
},
|
||||
(("collision-wild/*", "collision-wild-1", "openai/*"),),
|
||||
)
|
||||
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
|
||||
|
|
@ -211,6 +230,128 @@ def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the
|
|||
assert _served_by(received) == ("collision-b-1",)
|
||||
|
||||
|
||||
def _streaming(build: Build) -> Build:
|
||||
def streamed(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
path, body = build(model, marker)
|
||||
return path, {**body, "stream": True}
|
||||
|
||||
return streamed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build",
|
||||
(
|
||||
pytest.param(_streaming(_chat), id="chat-completions-stream"),
|
||||
pytest.param(_streaming(_messages), id="anthropic-messages-stream"),
|
||||
pytest.param(_streaming(_responses), id="responses-stream"),
|
||||
),
|
||||
)
|
||||
def test_a_streaming_request_for_a_shared_raw_model_string_is_refused_before_any_upstream_call(
|
||||
guarded: Gateway, peer: Wire, build: Build
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
peer.drain()
|
||||
refused: Final = guarded.request("POST", *build(SHARED_MODEL, uuid.uuid4().hex), key=scenario.key())
|
||||
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert SHARED_MODEL in refused.text
|
||||
assert peer.drain() == ()
|
||||
|
||||
|
||||
def _v1(gateway: Gateway) -> str:
|
||||
return f"{str(gateway.client.base_url).rstrip('/')}/v1"
|
||||
|
||||
|
||||
def _origin(gateway: Gateway) -> str:
|
||||
return str(gateway.client.base_url).rstrip("/")
|
||||
|
||||
|
||||
def _openai_chat(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.chat.completions.create(model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}])
|
||||
|
||||
|
||||
def _openai_responses(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.responses.create(model=SHARED_MODEL, input="sdk")
|
||||
|
||||
|
||||
def _openai_embeddings(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.embeddings.create(model=SHARED_MODEL, input=["sdk"])
|
||||
|
||||
|
||||
def _run(call: Callable[[], Awaitable[object]]) -> None:
|
||||
asyncio.run(call())
|
||||
|
||||
|
||||
def _async_openai_chat(gateway: Gateway, key: str) -> None:
|
||||
async def call() -> object:
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
|
||||
openai.AsyncOpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
return await client.chat.completions.create(
|
||||
model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}]
|
||||
)
|
||||
|
||||
_run(call)
|
||||
|
||||
|
||||
def _anthropic_messages(gateway: Gateway, key: str) -> None:
|
||||
with (
|
||||
httpx.Client(timeout=15, trust_env=False) as transport,
|
||||
anthropic.Anthropic(api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport) as client,
|
||||
):
|
||||
client.messages.create(model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}])
|
||||
|
||||
|
||||
def _async_anthropic_messages(gateway: Gateway, key: str) -> None:
|
||||
async def call() -> object:
|
||||
async with (
|
||||
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
|
||||
anthropic.AsyncAnthropic(
|
||||
api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport
|
||||
) as client,
|
||||
):
|
||||
return await client.messages.create(
|
||||
model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}]
|
||||
)
|
||||
|
||||
_run(call)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call", "error"),
|
||||
(
|
||||
pytest.param(_openai_chat, openai.BadRequestError, id="openai-sync-chat"),
|
||||
pytest.param(_async_openai_chat, openai.BadRequestError, id="openai-async-chat"),
|
||||
pytest.param(_openai_responses, openai.BadRequestError, id="openai-sync-responses"),
|
||||
pytest.param(_openai_embeddings, openai.BadRequestError, id="openai-sync-embeddings"),
|
||||
pytest.param(_anthropic_messages, anthropic.BadRequestError, id="anthropic-sync-messages"),
|
||||
pytest.param(_async_anthropic_messages, anthropic.BadRequestError, id="anthropic-async-messages"),
|
||||
),
|
||||
)
|
||||
def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string(
|
||||
guarded: Gateway, peer: Wire, call: SdkCall, error: type[Exception]
|
||||
) -> None:
|
||||
with guarded.scenario() as scenario:
|
||||
peer.drain()
|
||||
|
||||
with pytest.raises(error, match="is the litellm_params.model of more than one model group"):
|
||||
call(guarded, scenario.key())
|
||||
assert peer.drain() == ()
|
||||
|
||||
|
||||
def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on(
|
||||
guarded: Gateway, peer: Wire
|
||||
) -> None:
|
||||
|
|
@ -357,3 +498,71 @@ def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart
|
|||
assert refused.status_code == 400, refused.text
|
||||
assert after.status_code == 200, after.text
|
||||
assert len(served) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "enabled"),
|
||||
(
|
||||
pytest.param("TRUE", True, id="upper-case-true"),
|
||||
pytest.param("yes", False, id="yes"),
|
||||
pytest.param(1, False, id="integer-one"),
|
||||
pytest.param("", False, id="empty-string"),
|
||||
),
|
||||
)
|
||||
def test_only_a_true_value_from_the_admin_ui_turns_the_check_on(
|
||||
toggled: Gateway, peer: Wire, value: JsonValue, enabled: bool
|
||||
) -> None:
|
||||
with toggled.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
scenario.cleanups.callback(
|
||||
toggled.post, "/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}}
|
||||
)
|
||||
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": value}})
|
||||
peer.drain()
|
||||
response: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
|
||||
served: Final = _served_by(peer.drain())
|
||||
|
||||
assert response.status_code == (400 if enabled else 200), response.text
|
||||
assert len(served) == (0 if enabled else 1)
|
||||
|
||||
|
||||
def _burst_call(gateway: Gateway, key: str, index: int) -> tuple[bool, int | None, str]:
|
||||
raw: Final = index % 2 == 0
|
||||
build: Final = (_chat, _messages, _responses)[index // 2 % 3]
|
||||
marker: Final = f"collision-burst-{index}-{uuid.uuid4().hex}"
|
||||
path, body = build(SHARED_MODEL if raw else "collision-chain-b", marker)
|
||||
try:
|
||||
response: Final = gateway.request("POST", path, {**body, "stream": raw and index % 4 == 0}, key=key)
|
||||
except httpx.HTTPError:
|
||||
return raw, None, marker
|
||||
return raw, response.status_code, marker
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 180)
|
||||
def test_the_refusal_holds_on_both_workers_through_a_worker_kill_mid_burst(peer: Wire, tmp_path: Path) -> None:
|
||||
path: Final = _config(peer, tmp_path, {"enable_cross_model_group_collision_check": True})
|
||||
with (
|
||||
gateway_from_environment() as base,
|
||||
owned_proxy_process(base, tmp_path, {}, config=path, workers=2) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
key: Final = scenario.key()
|
||||
workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30)
|
||||
peer.drain()
|
||||
with ThreadPoolExecutor(max_workers=BURST) as pool:
|
||||
futures: Final = [pool.submit(_burst_call, owned.gateway, key, index) for index in range(BURST)]
|
||||
with owned.replacing_worker(min(workers)) as kill:
|
||||
kill()
|
||||
outcomes: Final = [future.result() for future in futures]
|
||||
upstream_bodies: Final = tuple(request.body.decode() for request in peer.drain())
|
||||
probes: Final = tuple(
|
||||
owned.gateway.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
|
||||
)
|
||||
control: Final = owned.gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key)
|
||||
|
||||
assert all(status in (400, None) for raw, status, _ in outcomes if raw), outcomes
|
||||
assert any(status == 400 for raw, status, _ in outcomes if raw), outcomes
|
||||
assert all(status in (200, None) for raw, status, _ in outcomes if not raw), outcomes
|
||||
assert not any(marker in body for raw, _, marker in outcomes if raw for body in upstream_bodies)
|
||||
assert [probe.status_code for probe in probes] == [400] * 8, [probe.text for probe in probes]
|
||||
assert control.status_code == 200, control.text
|
||||
|
|
|
|||
|
|
@ -589,6 +589,45 @@ def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_wit
|
|||
assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1"
|
||||
|
||||
|
||||
def _served_model_list() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
**deployment,
|
||||
"litellm_params": {
|
||||
**deployment["litellm_params"],
|
||||
"mock_response": f"served by {deployment['model_info']['id']}",
|
||||
},
|
||||
}
|
||||
for deployment in _COLLIDING_MODEL_LIST
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fallbacks",
|
||||
(
|
||||
pytest.param([{"*": ["solo"]}], id="wildcard-fallback"),
|
||||
pytest.param([{"alias": ["solo"]}, {_SHARED_LITELLM_MODEL: ["solo"]}], id="keyed-fallback"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("model", ("alias", _SHARED_LITELLM_MODEL))
|
||||
def test_collision_check_refusal_is_not_rescued_by_a_fallback(
|
||||
fallbacks: list[dict[str, list[str]]], model: str
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=_served_model_list(),
|
||||
model_group_alias={"alias": _SHARED_LITELLM_MODEL},
|
||||
fallbacks=fallbacks,
|
||||
enable_cross_model_group_collision_check=True,
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
router.completion(model=model, messages=messages)
|
||||
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
|
||||
asyncio.run(router.acompletion(model=model, messages=messages))
|
||||
assert router.completion(model="chain-b", messages=messages).choices[0].message.content == "served by chain-b-1" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # completion returns an untyped response union
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "enabled"),
|
||||
((True, True), ("true", True), (False, False), ("false", False)),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue