diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..9ecf58cdd8a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -72,6 +72,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "model_group_alias", "enable_weighted_failover", "enable_mid_stream_fallback_continuation", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", "optional_pre_call_checks", diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..517c6fbe95d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + RawModelGroupCollisionError, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -220,6 +221,7 @@ from litellm.router_utils.common_utils import ( get_request_team_id, is_proxy_admin_request, provider_for_generic_call, + refuse_litellm_model_spanning_model_groups, request_user_api_key_auth, resolve_model_group_alias, team_may_use_deployment, @@ -1272,6 +1274,7 @@ class Router: fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, session_affinity_repin_on_failover: bool = False, + enable_cross_model_group_collision_check: bool = False, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -1315,6 +1318,7 @@ class Router: enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). + enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. Returns: Router: An instance of the litellm.Router class. @@ -1526,6 +1530,7 @@ class Router: self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation) + self.enable_cross_model_group_collision_check = self._is_flag_enabled(enable_cross_model_group_collision_check) self.health_check_ignore_transient_errors = health_check_ignore_transient_errors self.background_health_check_model_groups: frozenset[str] | None = ( frozenset(background_health_check_model_groups) @@ -8567,6 +8572,7 @@ class Router: or original_model_group is None or is_guardrail_intervention(e) or unsupported_params_without_fallback + or isinstance(e, RawModelGroupCollisionError) ): raise e @@ -13996,6 +14002,7 @@ class Router: "model_group_alias", "enable_weighted_failover", "enable_mid_stream_fallback_continuation", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", "fallback_on_unsupported_params", @@ -14053,9 +14060,13 @@ class Router: elif var == "optional_pre_call_checks": self.set_optional_pre_call_checks(kwargs[var]) elif var == "enable_mid_stream_fallback_continuation": - self._set_mid_stream_fallback_continuation(kwargs[var]) + self._set_mid_stream_fallback_continuation(cast(object, kwargs[var])) # cast-ok: untyped **kwargs + elif var == "enable_cross_model_group_collision_check": + self.enable_cross_model_group_collision_check = self._is_flag_enabled( + cast(object, kwargs[var]) # cast-ok: untyped **kwargs + ) elif var == "retry_policy": - self._set_retry_policy(kwargs[var]) + self._set_retry_policy(cast(object, kwargs[var])) # cast-ok: untyped **kwargs else: value = kwargs[var] # only run routing strategy init if it has changed @@ -14746,7 +14757,7 @@ class Router: request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: - return model, self._drop_strategy_markers( + specific_deployments: Final = self._drop_strategy_markers( model, self._filter_reserved_deployments( model=model, @@ -14754,6 +14765,11 @@ class Router: request_team_id=request_team_id, ), ) + if self.enable_cross_model_group_collision_check: + refuse_litellm_model_spanning_model_groups( + model, deployments_the_caller_may_use(specific_deployments, request_kwargs) + ) + return model, specific_deployments elif model not in self.model_names and self.has_model_id(model): deployment: Final = self.get_deployment(model_id=model) if deployment is not None: @@ -14846,6 +14862,10 @@ class Router: ), request_team_id=request_team_id, ) + if self.enable_cross_model_group_collision_check: + refuse_litellm_model_spanning_model_groups( + model, deployments_the_caller_may_use(healthy_deployments, request_kwargs) + ) # If the litellm-model lookup produced candidates that access-group # filtering then removed, treat this the same as the by-name path # being emptied: prevent default-model fallback from bypassing the diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index a00d84babf3..e9eed6a9174 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -330,6 +330,42 @@ def filter_team_based_models( ] +def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]: + """A team deployment's model group is its team and public name, with no name for a team deployment created + before public names; any other deployment's is its `model_name`.""" + model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {}) + team_id: Final = model_info.get("team_id") + public_name: Final = model_info.get("team_public_model_name") + if isinstance(public_name, str) and public_name: + return team_id, public_name + model_name: Final = deployment["model_name"] + if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"): + return team_id, None + return None, model_name + + +class RawModelGroupCollisionError(BadRequestError): + """The 400 for a raw `litellm_params.model` that several model groups share; router fallbacks never serve it.""" + + +def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None: + """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group. + + A team with a matching deployment created before public names counts as one model group. + """ + groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments) + unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None) + counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups) + if len(counted) < 2: + return + raise RawModelGroupCollisionError( + message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. " + "Send one of the model group names instead", + model=model, + llm_provider="", + ) + + def deployments_the_caller_may_use( deployments: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] | None ) -> tuple[Mapping[str, object], ...]: diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index 64068f30a16..c103ca25cec 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -272,4 +272,13 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [ field_default=None, ui_field_name="Disable Cooldowns", ), + RouterSettingsField( + field_name="enable_cross_model_group_collision_check", + field_type="Boolean", + field_value=None, + field_description="Refuse a request for a litellm_params.model that deployments in more than one model " + "group share, instead of load balancing it across all of those groups", + field_default=False, + ui_field_name="Refuse Raw Models Shared Across Groups", + ), ] diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py new file mode 100644 index 00000000000..c91acc41a82 --- /dev/null +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -0,0 +1,637 @@ +import asyncio +import json +import uuid +from collections.abc import Awaitable, Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(240) + +SHARED_MODEL: Final = "openai/integration-shared-raw-model" +SOLO_MODEL: Final = "openai/integration-solo-raw-model" +DEPLOYMENTS: Final = ( + ("collision-chain-a", "collision-a-1", SHARED_MODEL), + ("collision-chain-a", "collision-a-2", SHARED_MODEL), + ("collision-chain-b", "collision-b-1", SHARED_MODEL), + ("collision-solo", "collision-solo-1", SOLO_MODEL), +) +CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"}) +TEAM_MODEL: Final = "openai/integration-team-raw-model" +TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"}) +TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names" +ALIAS: Final = "collision-alias" +POOLED_CALLS: Final = 48 +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +CATCH_ALL_FALLBACK: Final[dict[str, JsonValue]] = {"fallbacks": [{"*": ["collision-solo"]}]} + +Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]] +SdkCall = Callable[[Gateway, str], None] +BURST: Final = 30 + + +def _json_reply(body: dict[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _reply(request: Request) -> Reply: + if request.target.endswith("/chat/completions"): + return _json_reply( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "integration-shared-raw-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ) + if request.target.endswith("/responses"): + return _json_reply( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "integration-shared-raw-model", + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + ) + if request.target.endswith("/embeddings"): + return _json_reply( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "integration-shared-raw-model", + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } + ) + return Reply(status=404) + + +def _served_by(requests: tuple[Request, ...]) -> tuple[str, ...]: + return tuple(request.target.split("/")[1] for request in requests) + + +def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, JsonValue]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"}, + "model_info": {"id": identity}, + } + + +def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]: + return { + "model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}", + "litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"}, + "model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES}, + } + + +def _config( + wire: Wire, + directory: Path, + router_settings: dict[str, JsonValue], + extra_deployments: tuple[tuple[str, str, str], ...] = (), + team_deployments: tuple[dict[str, JsonValue], ...] = (), +) -> Path: + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config["model_list"] = [ + *(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)), + *team_deployments, + ] + config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings} + path: Final = directory / "raw-model-group-collision.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def peer() -> Iterator[Wire]: + with wire_server(_reply) as wire: + yield wire + + +@pytest.fixture(scope="module") +def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded") + path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True, **CATCH_ALL_FALLBACK}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def unguarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-default") + path: Final = _config(peer, directory, {}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-wildcard") + path: Final = _config( + peer, + directory, + { + "enable_cross_model_group_collision_check": True, + "model_group_alias": {ALIAS: SHARED_MODEL}, + **CATCH_ALL_FALLBACK, + }, + (("collision-wild/*", "collision-wild-1", "openai/*"),), + ) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def guarded_with_team_models_from_before_public_names( + peer: Wire, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names") + path: Final = _config( + peer, + directory, + {"enable_cross_model_group_collision_check": True}, + team_deployments=( + *(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS), + _team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL), + ), + ) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled") + path: Final = _config(peer, directory, {}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +def _chat(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + + +def _messages(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/messages", {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]} + + +def _responses(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/responses", {"model": model, "input": marker} + + +def _embeddings(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/embeddings", {"model": model, "input": [marker]} + + +@pytest.mark.parametrize( + "build", + ( + pytest.param(_chat, id="chat-completions"), + pytest.param(_messages, id="anthropic-messages"), + pytest.param(_responses, id="responses"), + pytest.param(_embeddings, id="embeddings"), + ), +) +def test_a_raw_model_string_shared_by_two_groups_is_refused_before_any_upstream_call( + guarded: Gateway, peer: Wire, build: Build +) -> None: + with guarded.scenario() as scenario: + key: Final = scenario.key() + refused_marker: Final = f"collision-refused-{uuid.uuid4().hex}" + control_marker: Final = f"collision-control-{uuid.uuid4().hex}" + peer.drain() + refused: Final = guarded.request("POST", *build(SHARED_MODEL, refused_marker), key=key) + control: Final = guarded.request("POST", *build("collision-chain-b", control_marker), key=key) + received: Final = peer.drain() + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert control.status_code == 200, control.text + assert _served_by(received) == ("collision-b-1",) + assert control_marker in received[0].body.decode() + assert refused_marker not in received[0].body.decode() + + +@pytest.mark.parametrize("model", (pytest.param(SHARED_MODEL, id="raw-model"), pytest.param(ALIAS, id="alias"))) +def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the_generic_lookup( + guarded_without_specific_deployment: Gateway, peer: Wire, model: str +) -> None: + gateway: Final = guarded_without_specific_deployment + with gateway.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + refused: Final = gateway.request("POST", *_chat(model, uuid.uuid4().hex), key=key) + control: Final = gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key) + received: Final = peer.drain() + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert control.status_code == 200, control.text + assert _served_by(received) == ("collision-b-1",) + + +def _streaming(build: Build) -> Build: + def streamed(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + path, body = build(model, marker) + return path, {**body, "stream": True} + + return streamed + + +@pytest.mark.parametrize( + "build", + ( + pytest.param(_streaming(_chat), id="chat-completions-stream"), + pytest.param(_streaming(_messages), id="anthropic-messages-stream"), + pytest.param(_streaming(_responses), id="responses-stream"), + ), +) +def test_a_streaming_request_for_a_shared_raw_model_string_is_refused_before_any_upstream_call( + guarded: Gateway, peer: Wire, build: Build +) -> None: + with guarded.scenario() as scenario: + peer.drain() + refused: Final = guarded.request("POST", *build(SHARED_MODEL, uuid.uuid4().hex), key=scenario.key()) + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert peer.drain() == () + + +def _v1(gateway: Gateway) -> str: + return f"{str(gateway.client.base_url).rstrip('/')}/v1" + + +def _origin(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _openai_chat(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.chat.completions.create(model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}]) + + +def _openai_responses(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.responses.create(model=SHARED_MODEL, input="sdk") + + +def _openai_embeddings(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.embeddings.create(model=SHARED_MODEL, input=["sdk"]) + + +def _run(call: Callable[[], Awaitable[object]]) -> None: + asyncio.run(call()) + + +def _async_openai_chat(gateway: Gateway, key: str) -> None: + async def call() -> object: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + openai.AsyncOpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + return await client.chat.completions.create( + model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}] + ) + + _run(call) + + +def _anthropic_messages(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + anthropic.Anthropic(api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport) as client, + ): + client.messages.create(model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}]) + + +def _async_anthropic_messages(gateway: Gateway, key: str) -> None: + async def call() -> object: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + anthropic.AsyncAnthropic( + api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + return await client.messages.create( + model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}] + ) + + _run(call) + + +@pytest.mark.parametrize( + ("call", "error"), + ( + pytest.param(_openai_chat, openai.BadRequestError, id="openai-sync-chat"), + pytest.param(_async_openai_chat, openai.BadRequestError, id="openai-async-chat"), + pytest.param(_openai_responses, openai.BadRequestError, id="openai-sync-responses"), + pytest.param(_openai_embeddings, openai.BadRequestError, id="openai-sync-embeddings"), + pytest.param(_anthropic_messages, anthropic.BadRequestError, id="anthropic-sync-messages"), + pytest.param(_async_anthropic_messages, anthropic.BadRequestError, id="anthropic-async-messages"), + ), +) +def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string( + guarded: Gateway, peer: Wire, call: SdkCall, error: type[Exception] +) -> None: + with guarded.scenario() as scenario: + peer.drain() + + with pytest.raises(error, match="more than one model group"): + call(guarded, scenario.key()) + assert peer.drain() == () + + +def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat("collision-chain-a", uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= CHAIN_A + + +def test_a_raw_model_string_used_by_one_group_is_still_served_when_the_check_is_on( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + peer.drain() + response: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=scenario.key()) + + assert response.status_code == 200, response.text + assert _served_by(peer.drain()) == ("collision-solo-1",) + + +def test_without_the_setting_a_shared_raw_model_string_keeps_pooling_every_group( + unguarded: Gateway, peer: Wire +) -> None: + with unguarded.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + responses: Final = tuple( + unguarded.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(POOLED_CALLS) + ) + served: Final = set(_served_by(peer.drain())) + + assert [response.status_code for response in responses] == [200] * POOLED_CALLS, [r.text for r in responses] + assert "collision-b-1" in served + assert served & CHAIN_A + assert served <= CHAIN_A | {"collision-b-1"} + + +def _team_deployment( + scenario: Scenario, wire: Wire, team_id: str, public_name: str, identity: str, model: str = TEAM_MODEL +) -> None: + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": public_name, + "litellm_params": { + "model": model, + "api_base": f"{wire.url}/{identity}/v1", + "api_key": "synthetic-key", + }, + "model_info": {"team_id": team_id}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + + +def test_the_deployments_of_one_team_model_count_as_one_model_group(guarded: Gateway, peer: Wire) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + public_name: Final = f"collision-team-model-{uuid.uuid4().hex}" + for identity in sorted(TEAM_DEPLOYMENTS): + _team_deployment(scenario, peer, team, public_name, identity) + key: Final = scenario.key(team_id=team) + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= TEAM_DEPLOYMENTS + + +def test_another_teams_model_on_the_same_raw_model_string_does_not_refuse_this_team( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + other_team: Final = scenario.team() + public_name: Final = f"collision-team-model-{uuid.uuid4().hex}" + for identity in sorted(TEAM_DEPLOYMENTS): + _team_deployment(scenario, peer, team, public_name, identity) + _team_deployment(scenario, peer, other_team, public_name, "collision-other-team-1") + key: Final = scenario.key(team_id=team) + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= TEAM_DEPLOYMENTS + + +def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_for_that_team( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + _team_deployment( + scenario, peer, team, f"collision-team-model-{uuid.uuid4().hex}", "collision-team-solo-1", SOLO_MODEL + ) + team_key: Final = scenario.key(team_id=team) + other_key: Final = scenario.key() + peer.drain() + refused: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key) + served: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key) + received: Final = _served_by(peer.drain()) + + assert refused.status_code == 400, refused.text + assert SOLO_MODEL in refused.text + assert served.status_code == 200, served.text + assert received == ("collision-solo-1",) + + +def test_a_team_model_created_before_public_names_counts_as_one_model_group( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + _team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3") + key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + peer.drain() + responses: Final = tuple( + gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses] + assert len(served) == 12 + assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"} + + +def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + other_key: Final = scenario.key() + peer.drain() + refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key) + served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key) + received: Final = _served_by(peer.drain()) + + assert refused.status_code == 400, refused.text + assert SOLO_MODEL in refused.text + assert served.status_code == 200, served.text + assert received == ("collision-solo-1",) + + +def _router_setting(gateway: Gateway, name: str) -> JsonValue: + fields: Final = gateway.get("/router/settings")["fields"] + assert isinstance(fields, list) + return next(field["field_value"] for field in fields if isinstance(field, dict) and field["field_name"] == name) + + +def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart(toggled: Gateway, peer: Wire) -> None: + with toggled.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + before: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": True}}) + enabled_value: Final = _router_setting(toggled, "enable_cross_model_group_collision_check") + refused: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}}) + after: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + served: Final = _served_by(peer.drain()) + + assert before.status_code == 200, before.text + assert enabled_value is True + assert refused.status_code == 400, refused.text + assert after.status_code == 200, after.text + assert len(served) == 2 + + +@pytest.mark.parametrize( + ("value", "enabled"), + ( + pytest.param("TRUE", True, id="upper-case-true"), + pytest.param("yes", False, id="yes"), + pytest.param(1, False, id="integer-one"), + pytest.param("", False, id="empty-string"), + ), +) +def test_only_a_true_value_from_the_admin_ui_turns_the_check_on( + toggled: Gateway, peer: Wire, value: JsonValue, enabled: bool +) -> None: + with toggled.scenario() as scenario: + key: Final = scenario.key() + scenario.cleanups.callback( + toggled.post, "/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}} + ) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": value}}) + peer.drain() + response: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + served: Final = _served_by(peer.drain()) + + assert response.status_code == (400 if enabled else 200), response.text + assert len(served) == (0 if enabled else 1) + + +def _burst_call(gateway: Gateway, key: str, index: int) -> tuple[bool, int | None, str]: + raw: Final = index % 2 == 0 + build: Final = (_chat, _messages, _responses)[index // 2 % 3] + marker: Final = f"collision-burst-{index}-{uuid.uuid4().hex}" + path, body = build(SHARED_MODEL if raw else "collision-chain-b", marker) + try: + response: Final = gateway.request("POST", path, {**body, "stream": raw and index % 4 == 0}, key=key) + except httpx.HTTPError: + return raw, None, marker + return raw, response.status_code, marker + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 180) +def test_the_refusal_holds_on_both_workers_through_a_worker_kill_mid_burst(peer: Wire, tmp_path: Path) -> None: + path: Final = _config(peer, tmp_path, {"enable_cross_model_group_collision_check": True}) + with ( + gateway_from_environment() as base, + owned_proxy_process(base, tmp_path, {}, config=path, workers=2) as owned, + owned.gateway.scenario() as scenario, + ): + key: Final = scenario.key() + workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30) + peer.drain() + with ThreadPoolExecutor(max_workers=BURST) as pool: + futures: Final = [pool.submit(_burst_call, owned.gateway, key, index) for index in range(BURST)] + with owned.replacing_worker(min(workers)) as kill: + kill() + outcomes: Final = [future.result() for future in futures] + upstream_bodies: Final = tuple(request.body.decode() for request in peer.drain()) + probes: Final = tuple( + owned.gateway.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + control: Final = owned.gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key) + + assert all(status in (400, None) for raw, status, _ in outcomes if raw), outcomes + assert any(status == 400 for raw, status, _ in outcomes if raw), outcomes + assert all(status in (200, None) for raw, status, _ in outcomes if not raw), outcomes + assert not any(marker in body for raw, _, marker in outcomes if raw for body in upstream_bodies) + assert [probe.status_code for probe in probes] == [400] * 8, [probe.text for probe in probes] + assert control.status_code == 200, control.text diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 3532870df1c..3cf2cde1d20 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import contextlib import logging from itertools import chain from typing import Dict, Final, List, Optional, Union @@ -17,13 +18,21 @@ from litellm.router_utils.common_utils import ( filter_web_search_deployments, format_fallback_outcome_message, provider_for_generic_call, + refuse_litellm_model_spanning_model_groups, resolve_model_group_alias, resolve_served_model, truncate_fallback_error_detail, PROVIDER_SCOPED_CREDENTIAL_PARAMS, warn_on_provider_credential_mismatch, ) -from litellm.types.router import Deployment, FallbackSkipReason, LiteLLM_Params, ModelInfo, SkippedFallbackTarget +from litellm.types.router import ( + Deployment, + DeploymentTypedDict, + FallbackSkipReason, + LiteLLM_Params, + ModelInfo, + SkippedFallbackTarget, +) from litellm.types.utils import ProviderSpecificModelInfo @@ -957,3 +966,79 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d assert len(capability_warnings) == 1, capability_warnings assert "db-model" in capability_warnings[0] assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"] + + +def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict: + model_info: Final[dict[str, str]] = {} + if team is not None: + model_info["team_id"] = team[0] + if team[1] is not None: + model_info["team_public_model_name"] = team[1] + return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info} + + +@pytest.mark.parametrize( + ("deployments", "refused"), + ( + pytest.param((_group_member("chain-a"), _group_member("chain-b")), True, id="two-groups"), + pytest.param((_group_member("chain-a"), _group_member("chain-a")), False, id="one-group"), + pytest.param((_group_member("chain-a"),), False, id="one-deployment"), + pytest.param((), False, id="no-deployments"), + pytest.param( + ( + _group_member("model_name_t1_a", ("t1", "gpt-team")), + _group_member("model_name_t1_b", ("t1", "gpt-team")), + ), + False, + id="one-team-model", + ), + pytest.param( + ( + _group_member("model_name_t1_a", ("t1", "gpt-team")), + _group_member("model_name_t2_a", ("t2", "gpt-team")), + ), + True, + id="two-teams-same-public-name", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", "gpt-team")), _group_member("gpt-team")), + True, + id="team-model-and-proxy-group", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))), + False, + id="team-model-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))), + False, + id="team-deployments-from-before-and-after-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))), + True, + id="two-teams-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")), + True, + id="team-model-created-before-public-names-and-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("chain-a")), + True, + id="team-deployment-named-like-a-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")), + False, + id="team-deployment-in-a-proxy-group", + ), + ), +) +def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups( + deployments: tuple[DeploymentTypedDict, ...], refused: bool +) -> None: + with pytest.raises(litellm.BadRequestError) if refused else contextlib.nullcontext(): + refuse_litellm_model_spanning_model_groups("openai/shared-backend", deployments) diff --git a/tests/unit/test_router/test_router_get_deployments.py b/tests/unit/test_router/test_router_get_deployments.py index 07486762a64..24fd5180f9b 100644 --- a/tests/unit/test_router/test_router_get_deployments.py +++ b/tests/unit/test_router/test_router_get_deployments.py @@ -12,6 +12,7 @@ import pytest import litellm from litellm import Router +from litellm.types.router import RoutingStrategyName @pytest.mark.asyncio @@ -425,3 +426,259 @@ def test_get_routable_deployments_returns_every_deployment_routing_can_pick( } assert routable_ids == expected_ids + + +_SHARED_LITELLM_MODEL: Final = "openai/shared-backend" +_COLLIDING_MODEL_LIST: Final = [ + { + "model_name": "chain-a", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-a-1"}, + }, + { + "model_name": "chain-a", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-a-2"}, + }, + { + "model_name": "chain-b", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-b-1"}, + }, + { + "model_name": "solo", + "litellm_params": {"model": "openai/solo-backend", "api_key": "test-key"}, + "model_info": {"id": "solo-1"}, + }, +] + + +def _pick_sync(router: Router, model: str, specific_deployment: bool) -> str: + return router.get_available_deployment(model=model, specific_deployment=specific_deployment)["model_info"]["id"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + + +def _pick_async(router: Router, model: str, specific_deployment: bool) -> str: + async def pick() -> str: + deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + model=model, request_kwargs={}, specific_deployment=specific_deployment + ) + return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict + + return asyncio.run(pick()) + + +_PICKS: Final = ( + pytest.param(_pick_sync, False, id="sync"), + pytest.param(_pick_sync, True, id="sync-specific-deployment"), + pytest.param(_pick_async, False, id="async"), + pytest.param(_pick_async, True, id="async-specific-deployment"), +) + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_refuses_a_litellm_model_shared_by_two_model_groups( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + pick(router, _SHARED_LITELLM_MODEL, specific_deployment) + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_off_keeps_pooling_a_shared_litellm_model_across_groups( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST) + + picked: Final = {pick(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(60)} + + assert picked == {"chain-a-1", "chain-a-2", "chain-b-1"} + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_still_serves_a_litellm_model_used_by_a_single_group( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + assert pick(router, "openai/solo-backend", specific_deployment) == "solo-1" + + +def test_collision_check_leaves_model_group_names_routing_inside_their_group() -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + picked: Final = {_pick_sync(router, "chain-a", False) for _ in range(40)} + + assert picked == {"chain-a-1", "chain-a-2"} + + +@pytest.mark.parametrize("enabled", (True, False)) +def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_model(enabled: bool) -> None: + router: Final = Router( + model_list=_COLLIDING_MODEL_LIST, + model_group_alias={"alias": _SHARED_LITELLM_MODEL}, + enable_cross_model_group_collision_check=enabled, + ) + + if enabled: + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_sync(router, "alias", False) + else: + assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"} + + +def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]: + model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id} + if public_name is not None: + model_info["team_public_model_name"] = public_name + return { + "model_name": f"model_name_{team_id}_{deployment_id}", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": model_info, + } + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_treats_the_deployments_of_one_team_model_as_one_model_group( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-1-b", "team-1")], + enable_cross_model_group_collision_check=True, + ) + + picked: Final = {_pick_sync(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(40)} + + assert picked == {"team-1-a", "team-1-b"} + + +def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> str: + async def pick() -> str: + deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + model=_SHARED_LITELLM_MODEL, + request_kwargs={"metadata": {"user_api_key_team_id": team_id}}, + specific_deployment=specific_deployment, + ) + return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict + + return asyncio.run(pick()) + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +@pytest.mark.parametrize( + "public_names", + ( + pytest.param((None, None), id="both-created-before-public-names"), + pytest.param((None, "team-model"), id="one-created-before-public-names"), + ), +) +def test_collision_check_serves_a_team_model_created_before_public_names( + public_names: tuple[str | None, str | None], specific_deployment: bool +) -> None: + router: Final = Router( + model_list=[ + _team_deployment("team-1-a", "team-1", public_names[0]), + _team_deployment("team-1-b", "team-1", public_names[1]), + ], + enable_cross_model_group_collision_check=True, + ) + + picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)} + + assert picked == {"team-1-a", "team-1-b"} + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-2-a", "team-2")], + enable_cross_model_group_collision_check=True, + ) + + assert _pick_for_team(router, "team-1", specific_deployment) == "team-1-a" + assert _pick_for_team(router, "team-2", specific_deployment) == "team-2-a" + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_with_a_proxy_group( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _COLLIDING_MODEL_LIST[2]], + enable_cross_model_group_collision_check=True, + ) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_for_team(router, "team-1", specific_deployment) + assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1" + + +def _served_model_list() -> list[dict[str, object]]: + return [ + { + **deployment, + "litellm_params": { + **deployment["litellm_params"], + "mock_response": f"served by {deployment['model_info']['id']}", + }, + } + for deployment in _COLLIDING_MODEL_LIST + ] + + +@pytest.mark.parametrize( + "fallbacks", + ( + pytest.param([{"*": ["solo"]}], id="wildcard-fallback"), + pytest.param([{"alias": ["solo"]}, {_SHARED_LITELLM_MODEL: ["solo"]}], id="keyed-fallback"), + ), +) +@pytest.mark.parametrize("model", ("alias", _SHARED_LITELLM_MODEL)) +def test_collision_check_refusal_is_not_rescued_by_a_fallback( + fallbacks: list[dict[str, list[str]]], model: str +) -> None: + router: Final = Router( + model_list=_served_model_list(), + model_group_alias={"alias": _SHARED_LITELLM_MODEL}, + fallbacks=fallbacks, + enable_cross_model_group_collision_check=True, + ) + messages: Final = [{"role": "user", "content": "hi"}] + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + router.completion(model=model, messages=messages) + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + asyncio.run(router.acompletion(model=model, messages=messages)) + assert router.completion(model="chain-b", messages=messages).choices[0].message.content == "served by chain-b-1" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # completion returns an untyped response union + + +@pytest.mark.parametrize( + ("setting", "enabled"), + ((True, True), ("true", True), (False, False), ("false", False)), +) +def test_collision_check_can_be_toggled_through_update_settings(setting: bool | str, enabled: bool) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=not enabled) + + router.update_settings(enable_cross_model_group_collision_check=setting) # pyright: ignore[reportUnknownMemberType] # update_settings takes untyped **kwargs + + assert len(router.get_routable_deployments(_SHARED_LITELLM_MODEL)) == (0 if enabled else 3) + assert router.get_settings()["enable_cross_model_group_collision_check"] is enabled + + +@pytest.mark.parametrize( + "routing_strategy", + ("simple-shuffle", "least-busy", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"), +) +def test_collision_check_refuses_under_every_routing_strategy(routing_strategy: RoutingStrategyName) -> None: + router: Final = Router( + model_list=_COLLIDING_MODEL_LIST, + routing_strategy=routing_strategy, + enable_cross_model_group_collision_check=True, + ) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_async(router, _SHARED_LITELLM_MODEL, False) + assert _pick_async(router, "chain-a", False) in {"chain-a-1", "chain-a-2"}