This commit is contained in:
yucheng-berri 2026-10-27 16:48:57 -07:00 • committed by GitHub
commit c98bf12656
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1049 additions and 4 deletions

View file

@ -72,6 +72,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_cross_model_group_collision_check",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",

View file

@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import (
)
from litellm.router_utils.common_utils import (
MID_STREAM_CONTINUATION_KWARG,
RawModelGroupCollisionError,
deployments_the_caller_may_use,
filter_team_based_models,
filter_web_search_deployments,
@ -220,6 +221,7 @@ from litellm.router_utils.common_utils import (
get_request_team_id,
is_proxy_admin_request,
provider_for_generic_call,
refuse_litellm_model_spanning_model_groups,
request_user_api_key_auth,
resolve_model_group_alias,
team_may_use_deployment,
@ -1272,6 +1274,7 @@ class Router:
fallback_budget_check: FallbackBudgetCheck | None = None,
auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
session_affinity_repin_on_failover: bool = False,
enable_cross_model_group_collision_check: bool = False,
) -> None:
"""
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
@ -1315,6 +1318,7 @@ class Router:
enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False.
fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False.
Returns:
Router: An instance of the litellm.Router class.
@ -1526,6 +1530,7 @@ class Router:
self.enable_health_check_routing = enable_health_check_routing
self.enable_weighted_failover = enable_weighted_failover
self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation)
self.enable_cross_model_group_collision_check = self._is_flag_enabled(enable_cross_model_group_collision_check)
self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
self.background_health_check_model_groups: frozenset[str] | None = (
frozenset(background_health_check_model_groups)
@ -8567,6 +8572,7 @@ class Router:
or original_model_group is None
or is_guardrail_intervention(e)
or unsupported_params_without_fallback
or isinstance(e, RawModelGroupCollisionError)
):
raise e
@ -13996,6 +14002,7 @@ class Router:
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_cross_model_group_collision_check",
"enable_tag_filtering",
"tag_routing_prefix",
"fallback_on_unsupported_params",
@ -14053,9 +14060,13 @@ class Router:
elif var == "optional_pre_call_checks":
self.set_optional_pre_call_checks(kwargs[var])
elif var == "enable_mid_stream_fallback_continuation":
self._set_mid_stream_fallback_continuation(kwargs[var])
self._set_mid_stream_fallback_continuation(cast(object, kwargs[var])) # cast-ok: untyped **kwargs
elif var == "enable_cross_model_group_collision_check":
self.enable_cross_model_group_collision_check = self._is_flag_enabled(
cast(object, kwargs[var]) # cast-ok: untyped **kwargs
)
elif var == "retry_policy":
self._set_retry_policy(kwargs[var])
self._set_retry_policy(cast(object, kwargs[var])) # cast-ok: untyped **kwargs
else:
value = kwargs[var]
# only run routing strategy init if it has changed
@ -14746,7 +14757,7 @@ class Router:
request_team_id: Final = get_request_team_id(request_kwargs)
# check if aliases set on litellm model alias map
if specific_deployment is True:
return model, self._drop_strategy_markers(
specific_deployments: Final = self._drop_strategy_markers(
model,
self._filter_reserved_deployments(
model=model,
@ -14754,6 +14765,11 @@ class Router:
request_team_id=request_team_id,
),
)
if self.enable_cross_model_group_collision_check:
refuse_litellm_model_spanning_model_groups(
model, deployments_the_caller_may_use(specific_deployments, request_kwargs)
)
return model, specific_deployments
elif model not in self.model_names and self.has_model_id(model):
deployment: Final = self.get_deployment(model_id=model)
if deployment is not None:
@ -14846,6 +14862,10 @@ class Router:
),
request_team_id=request_team_id,
)
if self.enable_cross_model_group_collision_check:
refuse_litellm_model_spanning_model_groups(
model, deployments_the_caller_may_use(healthy_deployments, request_kwargs)
)
# If the litellm-model lookup produced candidates that access-group
# filtering then removed, treat this the same as the by-name path
# being emptied: prevent default-model fallback from bypassing the

View file

@ -330,6 +330,42 @@ def filter_team_based_models(
]
def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]:
"""A team deployment's model group is its team and public name, with no name for a team deployment created
before public names; any other deployment's is its `model_name`."""
model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {})
team_id: Final = model_info.get("team_id")
public_name: Final = model_info.get("team_public_model_name")
if isinstance(public_name, str) and public_name:
return team_id, public_name
model_name: Final = deployment["model_name"]
if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"):
return team_id, None
return None, model_name
class RawModelGroupCollisionError(BadRequestError):
"""The 400 for a raw `litellm_params.model` that several model groups share; router fallbacks never serve it."""
def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None:
"""Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.
A team with a matching deployment created before public names counts as one model group.
"""
groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments)
unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None)
counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups)
if len(counted) < 2:
return
raise RawModelGroupCollisionError(
message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. "
"Send one of the model group names instead",
model=model,
llm_provider="",
)
def deployments_the_caller_may_use(
deployments: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] | None
) -> tuple[Mapping[str, object], ...]:

View file

@ -272,4 +272,13 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [
field_default=None,
ui_field_name="Disable Cooldowns",
),
RouterSettingsField(
field_name="enable_cross_model_group_collision_check",
field_type="Boolean",
field_value=None,
field_description="Refuse a request for a litellm_params.model that deployments in more than one model "
"group share, instead of load balancing it across all of those groups",
field_default=False,
ui_field_name="Refuse Raw Models Shared Across Groups",
),
]

View file

@ -0,0 +1,637 @@
import asyncio
import json
import uuid
from collections.abc import Awaitable, Callable, Iterator
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final
import anthropic
import httpx
import openai
import pytest
import yaml
from integration._support.client import (
Gateway,
Scenario,
eventually,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(240)
SHARED_MODEL: Final = "openai/integration-shared-raw-model"
SOLO_MODEL: Final = "openai/integration-solo-raw-model"
DEPLOYMENTS: Final = (
("collision-chain-a", "collision-a-1", SHARED_MODEL),
("collision-chain-a", "collision-a-2", SHARED_MODEL),
("collision-chain-b", "collision-b-1", SHARED_MODEL),
("collision-solo", "collision-solo-1", SOLO_MODEL),
)
CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"})
TEAM_MODEL: Final = "openai/integration-team-raw-model"
TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"})
TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names"
ALIAS: Final = "collision-alias"
POOLED_CALLS: Final = 48
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
CATCH_ALL_FALLBACK: Final[dict[str, JsonValue]] = {"fallbacks": [{"*": ["collision-solo"]}]}
Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]]
SdkCall = Callable[[Gateway, str], None]
BURST: Final = 30
def _json_reply(body: dict[str, JsonValue]) -> Reply:
return Reply(body=json.dumps(body).encode())
def _reply(request: Request) -> Reply:
if request.target.endswith("/chat/completions"):
return _json_reply(
{
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "integration-shared-raw-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
)
if request.target.endswith("/responses"):
return _json_reply(
{
"id": f"resp_{uuid.uuid4().hex}",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "integration-shared-raw-model",
"output": [
{
"type": "message",
"id": f"msg_{uuid.uuid4().hex}",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
"usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4},
}
)
if request.target.endswith("/embeddings"):
return _json_reply(
{
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"model": "integration-shared-raw-model",
"usage": {"prompt_tokens": 3, "total_tokens": 3},
}
)
return Reply(status=404)
def _served_by(requests: tuple[Request, ...]) -> tuple[str, ...]:
return tuple(request.target.split("/")[1] for request in requests)
def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, JsonValue]:
return {
"model_name": group,
"litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"},
"model_info": {"id": identity},
}
def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]:
return {
"model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}",
"litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"},
"model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES},
}
def _config(
wire: Wire,
directory: Path,
router_settings: dict[str, JsonValue],
extra_deployments: tuple[tuple[str, str, str], ...] = (),
team_deployments: tuple[dict[str, JsonValue], ...] = (),
) -> Path:
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
config["model_list"] = [
*(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)),
*team_deployments,
]
config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings}
path: Final = directory / "raw-model-group-collision.yaml"
path.write_text(yaml.safe_dump(config))
return path
@pytest.fixture(scope="module")
def peer() -> Iterator[Wire]:
with wire_server(_reply) as wire:
yield wire
@pytest.fixture(scope="module")
def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded")
path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True, **CATCH_ALL_FALLBACK})
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
@pytest.fixture(scope="module")
def unguarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-default")
path: Final = _config(peer, directory, {})
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
@pytest.fixture(scope="module")
def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-wildcard")
path: Final = _config(
peer,
directory,
{
"enable_cross_model_group_collision_check": True,
"model_group_alias": {ALIAS: SHARED_MODEL},
**CATCH_ALL_FALLBACK,
},
(("collision-wild/*", "collision-wild-1", "openai/*"),),
)
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
@pytest.fixture(scope="module")
def guarded_with_team_models_from_before_public_names(
peer: Wire, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names")
path: Final = _config(
peer,
directory,
{"enable_cross_model_group_collision_check": True},
team_deployments=(
*(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS),
_team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL),
),
)
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
@pytest.fixture(scope="module")
def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled")
path: Final = _config(peer, directory, {})
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
def _chat(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]}
def _messages(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/messages", {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]}
def _responses(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/responses", {"model": model, "input": marker}
def _embeddings(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/embeddings", {"model": model, "input": [marker]}
@pytest.mark.parametrize(
"build",
(
pytest.param(_chat, id="chat-completions"),
pytest.param(_messages, id="anthropic-messages"),
pytest.param(_responses, id="responses"),
pytest.param(_embeddings, id="embeddings"),
),
)
def test_a_raw_model_string_shared_by_two_groups_is_refused_before_any_upstream_call(
guarded: Gateway, peer: Wire, build: Build
) -> None:
with guarded.scenario() as scenario:
key: Final = scenario.key()
refused_marker: Final = f"collision-refused-{uuid.uuid4().hex}"
control_marker: Final = f"collision-control-{uuid.uuid4().hex}"
peer.drain()
refused: Final = guarded.request("POST", *build(SHARED_MODEL, refused_marker), key=key)
control: Final = guarded.request("POST", *build("collision-chain-b", control_marker), key=key)
received: Final = peer.drain()
assert refused.status_code == 400, refused.text
assert SHARED_MODEL in refused.text
assert control.status_code == 200, control.text
assert _served_by(received) == ("collision-b-1",)
assert control_marker in received[0].body.decode()
assert refused_marker not in received[0].body.decode()
@pytest.mark.parametrize("model", (pytest.param(SHARED_MODEL, id="raw-model"), pytest.param(ALIAS, id="alias")))
def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the_generic_lookup(
guarded_without_specific_deployment: Gateway, peer: Wire, model: str
) -> None:
gateway: Final = guarded_without_specific_deployment
with gateway.scenario() as scenario:
key: Final = scenario.key()
peer.drain()
refused: Final = gateway.request("POST", *_chat(model, uuid.uuid4().hex), key=key)
control: Final = gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key)
received: Final = peer.drain()
assert refused.status_code == 400, refused.text
assert SHARED_MODEL in refused.text
assert control.status_code == 200, control.text
assert _served_by(received) == ("collision-b-1",)
def _streaming(build: Build) -> Build:
def streamed(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]:
path, body = build(model, marker)
return path, {**body, "stream": True}
return streamed
@pytest.mark.parametrize(
"build",
(
pytest.param(_streaming(_chat), id="chat-completions-stream"),
pytest.param(_streaming(_messages), id="anthropic-messages-stream"),
pytest.param(_streaming(_responses), id="responses-stream"),
),
)
def test_a_streaming_request_for_a_shared_raw_model_string_is_refused_before_any_upstream_call(
guarded: Gateway, peer: Wire, build: Build
) -> None:
with guarded.scenario() as scenario:
peer.drain()
refused: Final = guarded.request("POST", *build(SHARED_MODEL, uuid.uuid4().hex), key=scenario.key())
assert refused.status_code == 400, refused.text
assert SHARED_MODEL in refused.text
assert peer.drain() == ()
def _v1(gateway: Gateway) -> str:
return f"{str(gateway.client.base_url).rstrip('/')}/v1"
def _origin(gateway: Gateway) -> str:
return str(gateway.client.base_url).rstrip("/")
def _openai_chat(gateway: Gateway, key: str) -> None:
with (
httpx.Client(timeout=15, trust_env=False) as transport,
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
):
client.chat.completions.create(model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}])
def _openai_responses(gateway: Gateway, key: str) -> None:
with (
httpx.Client(timeout=15, trust_env=False) as transport,
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
):
client.responses.create(model=SHARED_MODEL, input="sdk")
def _openai_embeddings(gateway: Gateway, key: str) -> None:
with (
httpx.Client(timeout=15, trust_env=False) as transport,
openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
):
client.embeddings.create(model=SHARED_MODEL, input=["sdk"])
def _run(call: Callable[[], Awaitable[object]]) -> None:
asyncio.run(call())
def _async_openai_chat(gateway: Gateway, key: str) -> None:
async def call() -> object:
async with (
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
openai.AsyncOpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client,
):
return await client.chat.completions.create(
model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}]
)
_run(call)
def _anthropic_messages(gateway: Gateway, key: str) -> None:
with (
httpx.Client(timeout=15, trust_env=False) as transport,
anthropic.Anthropic(api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport) as client,
):
client.messages.create(model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}])
def _async_anthropic_messages(gateway: Gateway, key: str) -> None:
async def call() -> object:
async with (
httpx.AsyncClient(timeout=15, trust_env=False) as transport,
anthropic.AsyncAnthropic(
api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport
) as client,
):
return await client.messages.create(
model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}]
)
_run(call)
@pytest.mark.parametrize(
("call", "error"),
(
pytest.param(_openai_chat, openai.BadRequestError, id="openai-sync-chat"),
pytest.param(_async_openai_chat, openai.BadRequestError, id="openai-async-chat"),
pytest.param(_openai_responses, openai.BadRequestError, id="openai-sync-responses"),
pytest.param(_openai_embeddings, openai.BadRequestError, id="openai-sync-embeddings"),
pytest.param(_anthropic_messages, anthropic.BadRequestError, id="anthropic-sync-messages"),
pytest.param(_async_anthropic_messages, anthropic.BadRequestError, id="anthropic-async-messages"),
),
)
def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string(
guarded: Gateway, peer: Wire, call: SdkCall, error: type[Exception]
) -> None:
with guarded.scenario() as scenario:
peer.drain()
with pytest.raises(error, match="more than one model group"):
call(guarded, scenario.key())
assert peer.drain() == ()
def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on(
guarded: Gateway, peer: Wire
) -> None:
with guarded.scenario() as scenario:
key: Final = scenario.key()
peer.drain()
responses: Final = tuple(
guarded.request("POST", *_chat("collision-chain-a", uuid.uuid4().hex), key=key) for _ in range(8)
)
served: Final = _served_by(peer.drain())
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
assert len(served) == 8
assert set(served) <= CHAIN_A
def test_a_raw_model_string_used_by_one_group_is_still_served_when_the_check_is_on(
guarded: Gateway, peer: Wire
) -> None:
with guarded.scenario() as scenario:
peer.drain()
response: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=scenario.key())
assert response.status_code == 200, response.text
assert _served_by(peer.drain()) == ("collision-solo-1",)
def test_without_the_setting_a_shared_raw_model_string_keeps_pooling_every_group(
unguarded: Gateway, peer: Wire
) -> None:
with unguarded.scenario() as scenario:
key: Final = scenario.key()
peer.drain()
responses: Final = tuple(
unguarded.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(POOLED_CALLS)
)
served: Final = set(_served_by(peer.drain()))
assert [response.status_code for response in responses] == [200] * POOLED_CALLS, [r.text for r in responses]
assert "collision-b-1" in served
assert served & CHAIN_A
assert served <= CHAIN_A | {"collision-b-1"}
def _team_deployment(
scenario: Scenario, wire: Wire, team_id: str, public_name: str, identity: str, model: str = TEAM_MODEL
) -> None:
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": public_name,
"litellm_params": {
"model": model,
"api_base": f"{wire.url}/{identity}/v1",
"api_key": "synthetic-key",
},
"model_info": {"team_id": team_id},
},
)
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
def test_the_deployments_of_one_team_model_count_as_one_model_group(guarded: Gateway, peer: Wire) -> None:
with guarded.scenario() as scenario:
team: Final = scenario.team()
public_name: Final = f"collision-team-model-{uuid.uuid4().hex}"
for identity in sorted(TEAM_DEPLOYMENTS):
_team_deployment(scenario, peer, team, public_name, identity)
key: Final = scenario.key(team_id=team)
peer.drain()
responses: Final = tuple(
guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
)
served: Final = _served_by(peer.drain())
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
assert len(served) == 8
assert set(served) <= TEAM_DEPLOYMENTS
def test_another_teams_model_on_the_same_raw_model_string_does_not_refuse_this_team(
guarded: Gateway, peer: Wire
) -> None:
with guarded.scenario() as scenario:
team: Final = scenario.team()
other_team: Final = scenario.team()
public_name: Final = f"collision-team-model-{uuid.uuid4().hex}"
for identity in sorted(TEAM_DEPLOYMENTS):
_team_deployment(scenario, peer, team, public_name, identity)
_team_deployment(scenario, peer, other_team, public_name, "collision-other-team-1")
key: Final = scenario.key(team_id=team)
peer.drain()
responses: Final = tuple(
guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
)
served: Final = _served_by(peer.drain())
assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses]
assert len(served) == 8
assert set(served) <= TEAM_DEPLOYMENTS
def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_for_that_team(
guarded: Gateway, peer: Wire
) -> None:
with guarded.scenario() as scenario:
team: Final = scenario.team()
_team_deployment(
scenario, peer, team, f"collision-team-model-{uuid.uuid4().hex}", "collision-team-solo-1", SOLO_MODEL
)
team_key: Final = scenario.key(team_id=team)
other_key: Final = scenario.key()
peer.drain()
refused: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key)
served: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key)
received: Final = _served_by(peer.drain())
assert refused.status_code == 400, refused.text
assert SOLO_MODEL in refused.text
assert served.status_code == 200, served.text
assert received == ("collision-solo-1",)
def test_a_team_model_created_before_public_names_counts_as_one_model_group(
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
) -> None:
gateway: Final = guarded_with_team_models_from_before_public_names
with gateway.scenario() as scenario:
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
_team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3")
key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
peer.drain()
responses: Final = tuple(
gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12)
)
served: Final = _served_by(peer.drain())
assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses]
assert len(served) == 12
assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"}
def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused(
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
) -> None:
gateway: Final = guarded_with_team_models_from_before_public_names
with gateway.scenario() as scenario:
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
other_key: Final = scenario.key()
peer.drain()
refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key)
served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key)
received: Final = _served_by(peer.drain())
assert refused.status_code == 400, refused.text
assert SOLO_MODEL in refused.text
assert served.status_code == 200, served.text
assert received == ("collision-solo-1",)
def _router_setting(gateway: Gateway, name: str) -> JsonValue:
fields: Final = gateway.get("/router/settings")["fields"]
assert isinstance(fields, list)
return next(field["field_value"] for field in fields if isinstance(field, dict) and field["field_name"] == name)
def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart(toggled: Gateway, peer: Wire) -> None:
with toggled.scenario() as scenario:
key: Final = scenario.key()
peer.drain()
before: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": True}})
enabled_value: Final = _router_setting(toggled, "enable_cross_model_group_collision_check")
refused: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}})
after: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
served: Final = _served_by(peer.drain())
assert before.status_code == 200, before.text
assert enabled_value is True
assert refused.status_code == 400, refused.text
assert after.status_code == 200, after.text
assert len(served) == 2
@pytest.mark.parametrize(
("value", "enabled"),
(
pytest.param("TRUE", True, id="upper-case-true"),
pytest.param("yes", False, id="yes"),
pytest.param(1, False, id="integer-one"),
pytest.param("", False, id="empty-string"),
),
)
def test_only_a_true_value_from_the_admin_ui_turns_the_check_on(
toggled: Gateway, peer: Wire, value: JsonValue, enabled: bool
) -> None:
with toggled.scenario() as scenario:
key: Final = scenario.key()
scenario.cleanups.callback(
toggled.post, "/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}}
)
toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": value}})
peer.drain()
response: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key)
served: Final = _served_by(peer.drain())
assert response.status_code == (400 if enabled else 200), response.text
assert len(served) == (0 if enabled else 1)
def _burst_call(gateway: Gateway, key: str, index: int) -> tuple[bool, int | None, str]:
raw: Final = index % 2 == 0
build: Final = (_chat, _messages, _responses)[index // 2 % 3]
marker: Final = f"collision-burst-{index}-{uuid.uuid4().hex}"
path, body = build(SHARED_MODEL if raw else "collision-chain-b", marker)
try:
response: Final = gateway.request("POST", path, {**body, "stream": raw and index % 4 == 0}, key=key)
except httpx.HTTPError:
return raw, None, marker
return raw, response.status_code, marker
@pytest.mark.timeout(2 * graceful_stop_seconds() + 180)
def test_the_refusal_holds_on_both_workers_through_a_worker_kill_mid_burst(peer: Wire, tmp_path: Path) -> None:
path: Final = _config(peer, tmp_path, {"enable_cross_model_group_collision_check": True})
with (
gateway_from_environment() as base,
owned_proxy_process(base, tmp_path, {}, config=path, workers=2) as owned,
owned.gateway.scenario() as scenario,
):
key: Final = scenario.key()
workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30)
peer.drain()
with ThreadPoolExecutor(max_workers=BURST) as pool:
futures: Final = [pool.submit(_burst_call, owned.gateway, key, index) for index in range(BURST)]
with owned.replacing_worker(min(workers)) as kill:
kill()
outcomes: Final = [future.result() for future in futures]
upstream_bodies: Final = tuple(request.body.decode() for request in peer.drain())
probes: Final = tuple(
owned.gateway.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(8)
)
control: Final = owned.gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key)
assert all(status in (400, None) for raw, status, _ in outcomes if raw), outcomes
assert any(status == 400 for raw, status, _ in outcomes if raw), outcomes
assert all(status in (200, None) for raw, status, _ in outcomes if not raw), outcomes
assert not any(marker in body for raw, _, marker in outcomes if raw for body in upstream_bodies)
assert [probe.status_code for probe in probes] == [400] * 8, [probe.text for probe in probes]
assert control.status_code == 200, control.text

View file

@ -1,3 +1,4 @@
import contextlib
import logging
from itertools import chain
from typing import Dict, Final, List, Optional, Union
@ -17,13 +18,21 @@ from litellm.router_utils.common_utils import (
filter_web_search_deployments,
format_fallback_outcome_message,
provider_for_generic_call,
refuse_litellm_model_spanning_model_groups,
resolve_model_group_alias,
resolve_served_model,
truncate_fallback_error_detail,
PROVIDER_SCOPED_CREDENTIAL_PARAMS,
warn_on_provider_credential_mismatch,
)
from litellm.types.router import Deployment, FallbackSkipReason, LiteLLM_Params, ModelInfo, SkippedFallbackTarget
from litellm.types.router import (
Deployment,
DeploymentTypedDict,
FallbackSkipReason,
LiteLLM_Params,
ModelInfo,
SkippedFallbackTarget,
)
from litellm.types.utils import ProviderSpecificModelInfo
@ -957,3 +966,79 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d
assert len(capability_warnings) == 1, capability_warnings
assert "db-model" in capability_warnings[0]
assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"]
def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict:
model_info: Final[dict[str, str]] = {}
if team is not None:
model_info["team_id"] = team[0]
if team[1] is not None:
model_info["team_public_model_name"] = team[1]
return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info}
@pytest.mark.parametrize(
("deployments", "refused"),
(
pytest.param((_group_member("chain-a"), _group_member("chain-b")), True, id="two-groups"),
pytest.param((_group_member("chain-a"), _group_member("chain-a")), False, id="one-group"),
pytest.param((_group_member("chain-a"),), False, id="one-deployment"),
pytest.param((), False, id="no-deployments"),
pytest.param(
(
_group_member("model_name_t1_a", ("t1", "gpt-team")),
_group_member("model_name_t1_b", ("t1", "gpt-team")),
),
False,
id="one-team-model",
),
pytest.param(
(
_group_member("model_name_t1_a", ("t1", "gpt-team")),
_group_member("model_name_t2_a", ("t2", "gpt-team")),
),
True,
id="two-teams-same-public-name",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", "gpt-team")), _group_member("gpt-team")),
True,
id="team-model-and-proxy-group",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))),
False,
id="team-model-created-before-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))),
False,
id="team-deployments-from-before-and-after-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))),
True,
id="two-teams-created-before-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")),
True,
id="team-model-created-before-public-names-and-proxy-group",
),
pytest.param(
(_group_member("gpt-team", ("t1", None)), _group_member("chain-a")),
True,
id="team-deployment-named-like-a-proxy-group",
),
pytest.param(
(_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")),
False,
id="team-deployment-in-a-proxy-group",
),
),
)
def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups(
deployments: tuple[DeploymentTypedDict, ...], refused: bool
) -> None:
with pytest.raises(litellm.BadRequestError) if refused else contextlib.nullcontext():
refuse_litellm_model_spanning_model_groups("openai/shared-backend", deployments)

View file

@ -12,6 +12,7 @@ import pytest
import litellm
from litellm import Router
from litellm.types.router import RoutingStrategyName
@pytest.mark.asyncio
@ -425,3 +426,259 @@ def test_get_routable_deployments_returns_every_deployment_routing_can_pick(
}
assert routable_ids == expected_ids
_SHARED_LITELLM_MODEL: Final = "openai/shared-backend"
_COLLIDING_MODEL_LIST: Final = [
{
"model_name": "chain-a",
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
"model_info": {"id": "chain-a-1"},
},
{
"model_name": "chain-a",
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
"model_info": {"id": "chain-a-2"},
},
{
"model_name": "chain-b",
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
"model_info": {"id": "chain-b-1"},
},
{
"model_name": "solo",
"litellm_params": {"model": "openai/solo-backend", "api_key": "test-key"},
"model_info": {"id": "solo-1"},
},
]
def _pick_sync(router: Router, model: str, specific_deployment: bool) -> str:
return router.get_available_deployment(model=model, specific_deployment=specific_deployment)["model_info"]["id"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
def _pick_async(router: Router, model: str, specific_deployment: bool) -> str:
async def pick() -> str:
deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
model=model, request_kwargs={}, specific_deployment=specific_deployment
)
return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict
return asyncio.run(pick())
_PICKS: Final = (
pytest.param(_pick_sync, False, id="sync"),
pytest.param(_pick_sync, True, id="sync-specific-deployment"),
pytest.param(_pick_async, False, id="async"),
pytest.param(_pick_async, True, id="async-specific-deployment"),
)
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
def test_collision_check_refuses_a_litellm_model_shared_by_two_model_groups(
pick: Callable[[Router, str, bool], str], specific_deployment: bool
) -> None:
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
pick(router, _SHARED_LITELLM_MODEL, specific_deployment)
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
def test_collision_check_off_keeps_pooling_a_shared_litellm_model_across_groups(
pick: Callable[[Router, str, bool], str], specific_deployment: bool
) -> None:
router: Final = Router(model_list=_COLLIDING_MODEL_LIST)
picked: Final = {pick(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(60)}
assert picked == {"chain-a-1", "chain-a-2", "chain-b-1"}
@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS)
def test_collision_check_still_serves_a_litellm_model_used_by_a_single_group(
pick: Callable[[Router, str, bool], str], specific_deployment: bool
) -> None:
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
assert pick(router, "openai/solo-backend", specific_deployment) == "solo-1"
def test_collision_check_leaves_model_group_names_routing_inside_their_group() -> None:
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True)
picked: Final = {_pick_sync(router, "chain-a", False) for _ in range(40)}
assert picked == {"chain-a-1", "chain-a-2"}
@pytest.mark.parametrize("enabled", (True, False))
def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_model(enabled: bool) -> None:
router: Final = Router(
model_list=_COLLIDING_MODEL_LIST,
model_group_alias={"alias": _SHARED_LITELLM_MODEL},
enable_cross_model_group_collision_check=enabled,
)
if enabled:
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
_pick_sync(router, "alias", False)
else:
assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"}
def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]:
model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id}
if public_name is not None:
model_info["team_public_model_name"] = public_name
return {
"model_name": f"model_name_{team_id}_{deployment_id}",
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
"model_info": model_info,
}
@pytest.mark.parametrize("specific_deployment", (True, False))
def test_collision_check_treats_the_deployments_of_one_team_model_as_one_model_group(
specific_deployment: bool,
) -> None:
router: Final = Router(
model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-1-b", "team-1")],
enable_cross_model_group_collision_check=True,
)
picked: Final = {_pick_sync(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(40)}
assert picked == {"team-1-a", "team-1-b"}
def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> str:
async def pick() -> str:
deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict
model=_SHARED_LITELLM_MODEL,
request_kwargs={"metadata": {"user_api_key_team_id": team_id}},
specific_deployment=specific_deployment,
)
return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict
return asyncio.run(pick())
@pytest.mark.parametrize("specific_deployment", (True, False))
@pytest.mark.parametrize(
"public_names",
(
pytest.param((None, None), id="both-created-before-public-names"),
pytest.param((None, "team-model"), id="one-created-before-public-names"),
),
)
def test_collision_check_serves_a_team_model_created_before_public_names(
public_names: tuple[str | None, str | None], specific_deployment: bool
) -> None:
router: Final = Router(
model_list=[
_team_deployment("team-1-a", "team-1", public_names[0]),
_team_deployment("team-1-b", "team-1", public_names[1]),
],
enable_cross_model_group_collision_check=True,
)
picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)}
assert picked == {"team-1-a", "team-1-b"}
@pytest.mark.parametrize("specific_deployment", (True, False))
def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model(
specific_deployment: bool,
) -> None:
router: Final = Router(
model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-2-a", "team-2")],
enable_cross_model_group_collision_check=True,
)
assert _pick_for_team(router, "team-1", specific_deployment) == "team-1-a"
assert _pick_for_team(router, "team-2", specific_deployment) == "team-2-a"
@pytest.mark.parametrize("specific_deployment", (True, False))
def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_with_a_proxy_group(
specific_deployment: bool,
) -> None:
router: Final = Router(
model_list=[_team_deployment("team-1-a", "team-1"), _COLLIDING_MODEL_LIST[2]],
enable_cross_model_group_collision_check=True,
)
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
_pick_for_team(router, "team-1", specific_deployment)
assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1"
def _served_model_list() -> list[dict[str, object]]:
return [
{
**deployment,
"litellm_params": {
**deployment["litellm_params"],
"mock_response": f"served by {deployment['model_info']['id']}",
},
}
for deployment in _COLLIDING_MODEL_LIST
]
@pytest.mark.parametrize(
"fallbacks",
(
pytest.param([{"*": ["solo"]}], id="wildcard-fallback"),
pytest.param([{"alias": ["solo"]}, {_SHARED_LITELLM_MODEL: ["solo"]}], id="keyed-fallback"),
),
)
@pytest.mark.parametrize("model", ("alias", _SHARED_LITELLM_MODEL))
def test_collision_check_refusal_is_not_rescued_by_a_fallback(
fallbacks: list[dict[str, list[str]]], model: str
) -> None:
router: Final = Router(
model_list=_served_model_list(),
model_group_alias={"alias": _SHARED_LITELLM_MODEL},
fallbacks=fallbacks,
enable_cross_model_group_collision_check=True,
)
messages: Final = [{"role": "user", "content": "hi"}]
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
router.completion(model=model, messages=messages)
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
asyncio.run(router.acompletion(model=model, messages=messages))
assert router.completion(model="chain-b", messages=messages).choices[0].message.content == "served by chain-b-1" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # completion returns an untyped response union
@pytest.mark.parametrize(
("setting", "enabled"),
((True, True), ("true", True), (False, False), ("false", False)),
)
def test_collision_check_can_be_toggled_through_update_settings(setting: bool | str, enabled: bool) -> None:
router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=not enabled)
router.update_settings(enable_cross_model_group_collision_check=setting) # pyright: ignore[reportUnknownMemberType] # update_settings takes untyped **kwargs
assert len(router.get_routable_deployments(_SHARED_LITELLM_MODEL)) == (0 if enabled else 3)
assert router.get_settings()["enable_cross_model_group_collision_check"] is enabled
@pytest.mark.parametrize(
"routing_strategy",
("simple-shuffle", "least-busy", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"),
)
def test_collision_check_refuses_under_every_routing_strategy(routing_strategy: RoutingStrategyName) -> None:
router: Final = Router(
model_list=_COLLIDING_MODEL_LIST,
routing_strategy=routing_strategy,
enable_cross_model_group_collision_check=True,
)
with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"):
_pick_async(router, _SHARED_LITELLM_MODEL, False)
assert _pick_async(router, "chain-a", False) in {"chain-a-1", "chain-a-2"}