From 21e41e004467be5516f37b57316591202dc4b270 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:33:42 -0700 Subject: [PATCH 1/4] feat(router): opt-in refusal of raw model strings shared by several model groups A request whose model matches no model_name but equals the litellm_params.model of deployments in more than one model group is load balanced across all of them. enable_cross_model_group_collision_check (router_settings, Router kwarg, Admin UI router settings) refuses that request with a 400 instead. Only the deployments the caller's team may use are counted, and the deployments of one team model count as one group. Off by default, so routing is unchanged unless an operator turns it on. --- litellm/constants.py | 1 + litellm/router.py | 24 +- litellm/router_utils/common_utils.py | 21 + .../router_settings_endpoints.py | 9 + .../routing/test_raw_model_group_collision.py | 359 ++++++++++++++++++ .../test_router_utils_common_utils.py | 53 ++- .../test_router_get_deployments.py | 191 ++++++++++ 7 files changed, 654 insertions(+), 4 deletions(-) create mode 100644 tests/integration/routing/test_raw_model_group_collision.py diff --git a/litellm/constants.py b/litellm/constants.py index c7673db6976..de77977b1b8 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -70,6 +70,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "model_group_alias", "enable_weighted_failover", "enable_mid_stream_fallback_continuation", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", "optional_pre_call_checks", diff --git a/litellm/router.py b/litellm/router.py index ba3b4d53725..c3a25c28464 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -213,6 +213,7 @@ from litellm.router_utils.common_utils import ( get_request_team_id, is_proxy_admin_request, provider_for_generic_call, + refuse_litellm_model_spanning_model_groups, request_user_api_key_auth, resolve_model_group_alias, team_may_use_deployment, @@ -1209,6 +1210,7 @@ class Router: background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, enable_mid_stream_fallback_continuation: bool = False, + enable_cross_model_group_collision_check: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, @@ -1251,6 +1253,7 @@ class Router: ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. + enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). Returns: @@ -1459,6 +1462,7 @@ class Router: self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation) + self.enable_cross_model_group_collision_check = self._is_flag_enabled(enable_cross_model_group_collision_check) self.health_check_ignore_transient_errors = health_check_ignore_transient_errors self.background_health_check_model_groups: frozenset[str] | None = ( frozenset(background_health_check_model_groups) @@ -13655,6 +13659,7 @@ class Router: "model_group_alias", "enable_weighted_failover", "enable_mid_stream_fallback_continuation", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", "fallback_on_unsupported_params", @@ -13712,9 +13717,13 @@ class Router: elif var == "optional_pre_call_checks": self.set_optional_pre_call_checks(kwargs[var]) elif var == "enable_mid_stream_fallback_continuation": - self._set_mid_stream_fallback_continuation(kwargs[var]) + self._set_mid_stream_fallback_continuation(cast(object, kwargs[var])) # cast-ok: untyped **kwargs + elif var == "enable_cross_model_group_collision_check": + self.enable_cross_model_group_collision_check = self._is_flag_enabled( + cast(object, kwargs[var]) # cast-ok: untyped **kwargs + ) elif var == "retry_policy": - self._set_retry_policy(kwargs[var]) + self._set_retry_policy(cast(object, kwargs[var])) # cast-ok: untyped **kwargs else: value = kwargs[var] # only run routing strategy init if it has changed @@ -14296,7 +14305,7 @@ class Router: request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: - return model, self._drop_strategy_markers( + specific_deployments: Final = self._drop_strategy_markers( model, self._filter_reserved_deployments( model=model, @@ -14304,6 +14313,11 @@ class Router: request_team_id=request_team_id, ), ) + if self.enable_cross_model_group_collision_check: + refuse_litellm_model_spanning_model_groups( + model, deployments_the_caller_may_use(specific_deployments, request_kwargs) + ) + return model, specific_deployments elif model not in self.model_names and self.has_model_id(model): deployment: Final = self.get_deployment(model_id=model) if deployment is not None: @@ -14396,6 +14410,10 @@ class Router: ), request_team_id=request_team_id, ) + if self.enable_cross_model_group_collision_check: + refuse_litellm_model_spanning_model_groups( + model, deployments_the_caller_may_use(healthy_deployments, request_kwargs) + ) # If the litellm-model lookup produced candidates that access-group # filtering then removed, treat this the same as the by-name path # being emptied: prevent default-model fallback from bypassing the diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index a00d84babf3..6056ea44daf 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -330,6 +330,27 @@ def filter_team_based_models( ] +def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]: + """A team deployment's model group is its team and public name; any other deployment's is its `model_name`.""" + model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {}) + public_name: Final = model_info.get("team_public_model_name") + if isinstance(public_name, str) and public_name: + return model_info.get("team_id"), public_name + return None, deployment["model_name"] + + +def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None: + """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.""" + if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2: + return + raise BadRequestError( + message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. " + "Send one of the model group names instead", + model=model, + llm_provider="", + ) + + def deployments_the_caller_may_use( deployments: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] | None ) -> tuple[Mapping[str, object], ...]: diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index 9c8fe446d72..61b6d16f6b0 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -272,4 +272,13 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [ field_default=None, ui_field_name="Disable Cooldowns", ), + RouterSettingsField( + field_name="enable_cross_model_group_collision_check", + field_type="Boolean", + field_value=None, + field_description="Refuse a request for a litellm_params.model that deployments in more than one model " + "group share, instead of load balancing it across all of those groups", + field_default=False, + ui_field_name="Refuse Raw Models Shared Across Groups", + ), ] diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py new file mode 100644 index 00000000000..d894f9f395e --- /dev/null +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -0,0 +1,359 @@ +import json +import uuid +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value, string_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(240) + +SHARED_MODEL: Final = "openai/integration-shared-raw-model" +SOLO_MODEL: Final = "openai/integration-solo-raw-model" +DEPLOYMENTS: Final = ( + ("collision-chain-a", "collision-a-1", SHARED_MODEL), + ("collision-chain-a", "collision-a-2", SHARED_MODEL), + ("collision-chain-b", "collision-b-1", SHARED_MODEL), + ("collision-solo", "collision-solo-1", SOLO_MODEL), +) +CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"}) +TEAM_MODEL: Final = "openai/integration-team-raw-model" +TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"}) +ALIAS: Final = "collision-alias" +POOLED_CALLS: Final = 48 +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + +Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]] + + +def _json_reply(body: dict[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _reply(request: Request) -> Reply: + if request.target.endswith("/chat/completions"): + return _json_reply( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "integration-shared-raw-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ) + if request.target.endswith("/responses"): + return _json_reply( + { + "id": f"resp_{uuid.uuid4().hex}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "integration-shared-raw-model", + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + ) + if request.target.endswith("/embeddings"): + return _json_reply( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "integration-shared-raw-model", + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } + ) + return Reply(status=404) + + +def _served_by(requests: tuple[Request, ...]) -> tuple[str, ...]: + return tuple(request.target.split("/")[1] for request in requests) + + +def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, JsonValue]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"}, + "model_info": {"id": identity}, + } + + +def _config( + wire: Wire, + directory: Path, + router_settings: dict[str, JsonValue], + extra_deployments: tuple[tuple[str, str, str], ...] = (), +) -> Path: + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config["model_list"] = [_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)] + config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings} + path: Final = directory / "raw-model-group-collision.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def peer() -> Iterator[Wire]: + with wire_server(_reply) as wire: + yield wire + + +@pytest.fixture(scope="module") +def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded") + path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def unguarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-default") + path: Final = _config(peer, directory, {}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-wildcard") + path: Final = _config( + peer, + directory, + {"enable_cross_model_group_collision_check": True, "model_group_alias": {ALIAS: SHARED_MODEL}}, + (("collision-wild/*", "collision-wild-1", "openai/*"),), + ) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled") + path: Final = _config(peer, directory, {}) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + +def _chat(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + + +def _messages(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/messages", {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]} + + +def _responses(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/responses", {"model": model, "input": marker} + + +def _embeddings(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/embeddings", {"model": model, "input": [marker]} + + +@pytest.mark.parametrize( + "build", + ( + pytest.param(_chat, id="chat-completions"), + pytest.param(_messages, id="anthropic-messages"), + pytest.param(_responses, id="responses"), + pytest.param(_embeddings, id="embeddings"), + ), +) +def test_a_raw_model_string_shared_by_two_groups_is_refused_before_any_upstream_call( + guarded: Gateway, peer: Wire, build: Build +) -> None: + with guarded.scenario() as scenario: + key: Final = scenario.key() + refused_marker: Final = f"collision-refused-{uuid.uuid4().hex}" + control_marker: Final = f"collision-control-{uuid.uuid4().hex}" + peer.drain() + refused: Final = guarded.request("POST", *build(SHARED_MODEL, refused_marker), key=key) + control: Final = guarded.request("POST", *build("collision-chain-b", control_marker), key=key) + received: Final = peer.drain() + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert control.status_code == 200, control.text + assert _served_by(received) == ("collision-b-1",) + assert control_marker in received[0].body.decode() + assert refused_marker not in received[0].body.decode() + + +@pytest.mark.parametrize("model", (pytest.param(SHARED_MODEL, id="raw-model"), pytest.param(ALIAS, id="alias"))) +def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the_generic_lookup( + guarded_without_specific_deployment: Gateway, peer: Wire, model: str +) -> None: + gateway: Final = guarded_without_specific_deployment + with gateway.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + refused: Final = gateway.request("POST", *_chat(model, uuid.uuid4().hex), key=key) + control: Final = gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key) + received: Final = peer.drain() + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert control.status_code == 200, control.text + assert _served_by(received) == ("collision-b-1",) + + +def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat("collision-chain-a", uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= CHAIN_A + + +def test_a_raw_model_string_used_by_one_group_is_still_served_when_the_check_is_on( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + peer.drain() + response: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=scenario.key()) + + assert response.status_code == 200, response.text + assert _served_by(peer.drain()) == ("collision-solo-1",) + + +def test_without_the_setting_a_shared_raw_model_string_keeps_pooling_every_group( + unguarded: Gateway, peer: Wire +) -> None: + with unguarded.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + responses: Final = tuple( + unguarded.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(POOLED_CALLS) + ) + served: Final = set(_served_by(peer.drain())) + + assert [response.status_code for response in responses] == [200] * POOLED_CALLS, [r.text for r in responses] + assert "collision-b-1" in served + assert served & CHAIN_A + assert served <= CHAIN_A | {"collision-b-1"} + + +def _team_deployment( + scenario: Scenario, wire: Wire, team_id: str, public_name: str, identity: str, model: str = TEAM_MODEL +) -> None: + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": public_name, + "litellm_params": { + "model": model, + "api_base": f"{wire.url}/{identity}/v1", + "api_key": "synthetic-key", + }, + "model_info": {"team_id": team_id}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + + +def test_the_deployments_of_one_team_model_count_as_one_model_group(guarded: Gateway, peer: Wire) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + public_name: Final = f"collision-team-model-{uuid.uuid4().hex}" + for identity in sorted(TEAM_DEPLOYMENTS): + _team_deployment(scenario, peer, team, public_name, identity) + key: Final = scenario.key(team_id=team) + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= TEAM_DEPLOYMENTS + + +def test_another_teams_model_on_the_same_raw_model_string_does_not_refuse_this_team( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + other_team: Final = scenario.team() + public_name: Final = f"collision-team-model-{uuid.uuid4().hex}" + for identity in sorted(TEAM_DEPLOYMENTS): + _team_deployment(scenario, peer, team, public_name, identity) + _team_deployment(scenario, peer, other_team, public_name, "collision-other-team-1") + key: Final = scenario.key(team_id=team) + peer.drain() + responses: Final = tuple( + guarded.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 8, [r.text for r in responses] + assert len(served) == 8 + assert set(served) <= TEAM_DEPLOYMENTS + + +def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_for_that_team( + guarded: Gateway, peer: Wire +) -> None: + with guarded.scenario() as scenario: + team: Final = scenario.team() + _team_deployment( + scenario, peer, team, f"collision-team-model-{uuid.uuid4().hex}", "collision-team-solo-1", SOLO_MODEL + ) + team_key: Final = scenario.key(team_id=team) + other_key: Final = scenario.key() + peer.drain() + refused: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key) + served: Final = guarded.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key) + received: Final = _served_by(peer.drain()) + + assert refused.status_code == 400, refused.text + assert SOLO_MODEL in refused.text + assert served.status_code == 200, served.text + assert received == ("collision-solo-1",) + + +def _router_setting(gateway: Gateway, name: str) -> JsonValue: + fields: Final = gateway.get("/router/settings")["fields"] + assert isinstance(fields, list) + return next(field["field_value"] for field in fields if isinstance(field, dict) and field["field_name"] == name) + + +def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart(toggled: Gateway, peer: Wire) -> None: + with toggled.scenario() as scenario: + key: Final = scenario.key() + peer.drain() + before: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": True}}) + enabled_value: Final = _router_setting(toggled, "enable_cross_model_group_collision_check") + refused: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}}) + after: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + served: Final = _served_by(peer.drain()) + + assert before.status_code == 200, before.text + assert enabled_value is True + assert refused.status_code == 400, refused.text + assert after.status_code == 200, after.text + assert len(served) == 2 diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 3532870df1c..d8d07835295 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import contextlib import logging from itertools import chain from typing import Dict, Final, List, Optional, Union @@ -17,13 +18,21 @@ from litellm.router_utils.common_utils import ( filter_web_search_deployments, format_fallback_outcome_message, provider_for_generic_call, + refuse_litellm_model_spanning_model_groups, resolve_model_group_alias, resolve_served_model, truncate_fallback_error_detail, PROVIDER_SCOPED_CREDENTIAL_PARAMS, warn_on_provider_credential_mismatch, ) -from litellm.types.router import Deployment, FallbackSkipReason, LiteLLM_Params, ModelInfo, SkippedFallbackTarget +from litellm.types.router import ( + Deployment, + DeploymentTypedDict, + FallbackSkipReason, + LiteLLM_Params, + ModelInfo, + SkippedFallbackTarget, +) from litellm.types.utils import ProviderSpecificModelInfo @@ -957,3 +966,45 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d assert len(capability_warnings) == 1, capability_warnings assert "db-model" in capability_warnings[0] assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"] + + +def _group_member(model_name: str, team: tuple[str, str] | None = None) -> DeploymentTypedDict: + model_info: Final[dict[str, str]] = {} if team is None else {"team_id": team[0], "team_public_model_name": team[1]} + return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info} + + +@pytest.mark.parametrize( + ("deployments", "refused"), + ( + pytest.param((_group_member("chain-a"), _group_member("chain-b")), True, id="two-groups"), + pytest.param((_group_member("chain-a"), _group_member("chain-a")), False, id="one-group"), + pytest.param((_group_member("chain-a"),), False, id="one-deployment"), + pytest.param((), False, id="no-deployments"), + pytest.param( + ( + _group_member("model_name_t1_a", ("t1", "gpt-team")), + _group_member("model_name_t1_b", ("t1", "gpt-team")), + ), + False, + id="one-team-model", + ), + pytest.param( + ( + _group_member("model_name_t1_a", ("t1", "gpt-team")), + _group_member("model_name_t2_a", ("t2", "gpt-team")), + ), + True, + id="two-teams-same-public-name", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", "gpt-team")), _group_member("gpt-team")), + True, + id="team-model-and-proxy-group", + ), + ), +) +def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups( + deployments: tuple[DeploymentTypedDict, ...], refused: bool +) -> None: + with pytest.raises(litellm.BadRequestError) if refused else contextlib.nullcontext(): + refuse_litellm_model_spanning_model_groups("openai/shared-backend", deployments) diff --git a/tests/unit/test_router/test_router_get_deployments.py b/tests/unit/test_router/test_router_get_deployments.py index 07486762a64..9aaa037737d 100644 --- a/tests/unit/test_router/test_router_get_deployments.py +++ b/tests/unit/test_router/test_router_get_deployments.py @@ -12,6 +12,7 @@ import pytest import litellm from litellm import Router +from litellm.types.router import RoutingStrategyName @pytest.mark.asyncio @@ -425,3 +426,193 @@ def test_get_routable_deployments_returns_every_deployment_routing_can_pick( } assert routable_ids == expected_ids + + +_SHARED_LITELLM_MODEL: Final = "openai/shared-backend" +_COLLIDING_MODEL_LIST: Final = [ + { + "model_name": "chain-a", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-a-1"}, + }, + { + "model_name": "chain-a", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-a-2"}, + }, + { + "model_name": "chain-b", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": "chain-b-1"}, + }, + { + "model_name": "solo", + "litellm_params": {"model": "openai/solo-backend", "api_key": "test-key"}, + "model_info": {"id": "solo-1"}, + }, +] + + +def _pick_sync(router: Router, model: str, specific_deployment: bool) -> str: + return router.get_available_deployment(model=model, specific_deployment=specific_deployment)["model_info"]["id"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + + +def _pick_async(router: Router, model: str, specific_deployment: bool) -> str: + async def pick() -> str: + deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + model=model, request_kwargs={}, specific_deployment=specific_deployment + ) + return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict + + return asyncio.run(pick()) + + +_PICKS: Final = ( + pytest.param(_pick_sync, False, id="sync"), + pytest.param(_pick_sync, True, id="sync-specific-deployment"), + pytest.param(_pick_async, False, id="async"), + pytest.param(_pick_async, True, id="async-specific-deployment"), +) + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_refuses_a_litellm_model_shared_by_two_model_groups( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + pick(router, _SHARED_LITELLM_MODEL, specific_deployment) + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_off_keeps_pooling_a_shared_litellm_model_across_groups( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST) + + picked: Final = {pick(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(60)} + + assert picked == {"chain-a-1", "chain-a-2", "chain-b-1"} + + +@pytest.mark.parametrize(("pick", "specific_deployment"), _PICKS) +def test_collision_check_still_serves_a_litellm_model_used_by_a_single_group( + pick: Callable[[Router, str, bool], str], specific_deployment: bool +) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + assert pick(router, "openai/solo-backend", specific_deployment) == "solo-1" + + +def test_collision_check_leaves_model_group_names_routing_inside_their_group() -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=True) + + picked: Final = {_pick_sync(router, "chain-a", False) for _ in range(40)} + + assert picked == {"chain-a-1", "chain-a-2"} + + +@pytest.mark.parametrize("enabled", (True, False)) +def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_model(enabled: bool) -> None: + router: Final = Router( + model_list=_COLLIDING_MODEL_LIST, + model_group_alias={"alias": _SHARED_LITELLM_MODEL}, + enable_cross_model_group_collision_check=enabled, + ) + + if enabled: + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_sync(router, "alias", False) + else: + assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"} + + +def _team_deployment(deployment_id: str, team_id: str) -> dict[str, object]: + return { + "model_name": f"model_name_{team_id}_{deployment_id}", + "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, + "model_info": {"id": deployment_id, "team_id": team_id, "team_public_model_name": "team-model"}, + } + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_treats_the_deployments_of_one_team_model_as_one_model_group( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-1-b", "team-1")], + enable_cross_model_group_collision_check=True, + ) + + picked: Final = {_pick_sync(router, _SHARED_LITELLM_MODEL, specific_deployment) for _ in range(40)} + + assert picked == {"team-1-a", "team-1-b"} + + +def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> str: + async def pick() -> str: + deployment: Final = await router.async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the router returns an untyped deployment dict + model=_SHARED_LITELLM_MODEL, + request_kwargs={"metadata": {"user_api_key_team_id": team_id}}, + specific_deployment=specific_deployment, + ) + return deployment["model_info"]["id"] # pyright: ignore[reportUnknownVariableType] # the router returns an untyped deployment dict + + return asyncio.run(pick()) + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _team_deployment("team-2-a", "team-2")], + enable_cross_model_group_collision_check=True, + ) + + assert _pick_for_team(router, "team-1", specific_deployment) == "team-1-a" + assert _pick_for_team(router, "team-2", specific_deployment) == "team-2-a" + + +@pytest.mark.parametrize("specific_deployment", (True, False)) +def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_with_a_proxy_group( + specific_deployment: bool, +) -> None: + router: Final = Router( + model_list=[_team_deployment("team-1-a", "team-1"), _COLLIDING_MODEL_LIST[2]], + enable_cross_model_group_collision_check=True, + ) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_for_team(router, "team-1", specific_deployment) + assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1" + + +@pytest.mark.parametrize( + ("setting", "enabled"), + ((True, True), ("true", True), (False, False), ("false", False)), +) +def test_collision_check_can_be_toggled_through_update_settings(setting: bool | str, enabled: bool) -> None: + router: Final = Router(model_list=_COLLIDING_MODEL_LIST, enable_cross_model_group_collision_check=not enabled) + + router.update_settings(enable_cross_model_group_collision_check=setting) # pyright: ignore[reportUnknownMemberType] # update_settings takes untyped **kwargs + + assert len(router.get_routable_deployments(_SHARED_LITELLM_MODEL)) == (0 if enabled else 3) + assert router.get_settings()["enable_cross_model_group_collision_check"] is enabled + + +@pytest.mark.parametrize( + "routing_strategy", + ("simple-shuffle", "least-busy", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"), +) +def test_collision_check_refuses_under_every_routing_strategy(routing_strategy: RoutingStrategyName) -> None: + router: Final = Router( + model_list=_COLLIDING_MODEL_LIST, + routing_strategy=routing_strategy, + enable_cross_model_group_collision_check=True, + ) + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + _pick_async(router, _SHARED_LITELLM_MODEL, False) + assert _pick_async(router, "chain-a", False) in {"chain-a-1", "chain-a-2"} From c5e5bfbab4e19ae39bb5f33b278e099373d45f3b Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:52:47 -0700 Subject: [PATCH 2/4] fix(router): keep the raw model collision refusal out of router fallbacks A configured fallback, including a catch-all, could serve a request the check had refused. The refusal is now its own BadRequestError subclass that the fallback handler re-raises. Adds the audit cells: streaming, OpenAI and Anthropic SDK clients, hostile setting values and a worker kill mid burst. --- litellm/router.py | 2 + litellm/router_utils/common_utils.py | 6 +- .../routing/test_raw_model_group_collision.py | 219 +++++++++++++++++- .../test_router_get_deployments.py | 39 ++++ 4 files changed, 260 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index c3a25c28464..2fd89d3e68c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -205,6 +205,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + RawModelGroupCollisionError, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -8294,6 +8295,7 @@ class Router: or original_model_group is None or is_guardrail_intervention(e) or unsupported_params_without_fallback + or isinstance(e, RawModelGroupCollisionError) ): raise e diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 6056ea44daf..585b0b9b633 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -339,11 +339,15 @@ def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]: return None, deployment["model_name"] +class RawModelGroupCollisionError(BadRequestError): + """The 400 for a raw `litellm_params.model` that several model groups share; router fallbacks never serve it.""" + + def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None: """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.""" if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2: return - raise BadRequestError( + raise RawModelGroupCollisionError( message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. " "Send one of the model group names instead", model=model, diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py index d894f9f395e..e9a12172739 100644 --- a/tests/integration/routing/test_raw_model_group_collision.py +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -1,13 +1,25 @@ +import asyncio import json import uuid -from collections.abc import Callable, Iterator +from collections.abc import Awaitable, Callable, Iterator +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Final +import anthropic +import httpx +import openai import pytest import yaml -from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value, string_value -from integration._support.process import owned_proxy +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -27,8 +39,11 @@ TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"}) ALIAS: Final = "collision-alias" POOLED_CALLS: Final = 48 JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +CATCH_ALL_FALLBACK: Final[dict[str, JsonValue]] = {"fallbacks": [{"*": ["collision-solo"]}]} Build = Callable[[str, str], tuple[str, dict[str, JsonValue]]] +SdkCall = Callable[[Gateway, str], None] +BURST: Final = 30 def _json_reply(body: dict[str, JsonValue]) -> Reply: @@ -114,7 +129,7 @@ def peer() -> Iterator[Wire]: @pytest.fixture(scope="module") def guarded(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: directory: Final = tmp_path_factory.mktemp("raw-model-collision-guarded") - path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True}) + path: Final = _config(peer, directory, {"enable_cross_model_group_collision_check": True, **CATCH_ALL_FALLBACK}) with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: yield proxy @@ -133,7 +148,11 @@ def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.Tem path: Final = _config( peer, directory, - {"enable_cross_model_group_collision_check": True, "model_group_alias": {ALIAS: SHARED_MODEL}}, + { + "enable_cross_model_group_collision_check": True, + "model_group_alias": {ALIAS: SHARED_MODEL}, + **CATCH_ALL_FALLBACK, + }, (("collision-wild/*", "collision-wild-1", "openai/*"),), ) with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: @@ -211,6 +230,128 @@ def test_the_check_also_refuses_when_a_wildcard_route_sends_the_request_down_the assert _served_by(received) == ("collision-b-1",) +def _streaming(build: Build) -> Build: + def streamed(model: str, marker: str) -> tuple[str, dict[str, JsonValue]]: + path, body = build(model, marker) + return path, {**body, "stream": True} + + return streamed + + +@pytest.mark.parametrize( + "build", + ( + pytest.param(_streaming(_chat), id="chat-completions-stream"), + pytest.param(_streaming(_messages), id="anthropic-messages-stream"), + pytest.param(_streaming(_responses), id="responses-stream"), + ), +) +def test_a_streaming_request_for_a_shared_raw_model_string_is_refused_before_any_upstream_call( + guarded: Gateway, peer: Wire, build: Build +) -> None: + with guarded.scenario() as scenario: + peer.drain() + refused: Final = guarded.request("POST", *build(SHARED_MODEL, uuid.uuid4().hex), key=scenario.key()) + + assert refused.status_code == 400, refused.text + assert SHARED_MODEL in refused.text + assert peer.drain() == () + + +def _v1(gateway: Gateway) -> str: + return f"{str(gateway.client.base_url).rstrip('/')}/v1" + + +def _origin(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _openai_chat(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.chat.completions.create(model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}]) + + +def _openai_responses(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.responses.create(model=SHARED_MODEL, input="sdk") + + +def _openai_embeddings(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + openai.OpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + client.embeddings.create(model=SHARED_MODEL, input=["sdk"]) + + +def _run(call: Callable[[], Awaitable[object]]) -> None: + asyncio.run(call()) + + +def _async_openai_chat(gateway: Gateway, key: str) -> None: + async def call() -> object: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + openai.AsyncOpenAI(api_key=key, base_url=_v1(gateway), max_retries=0, http_client=transport) as client, + ): + return await client.chat.completions.create( + model=SHARED_MODEL, messages=[{"role": "user", "content": "sdk"}] + ) + + _run(call) + + +def _anthropic_messages(gateway: Gateway, key: str) -> None: + with ( + httpx.Client(timeout=15, trust_env=False) as transport, + anthropic.Anthropic(api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport) as client, + ): + client.messages.create(model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}]) + + +def _async_anthropic_messages(gateway: Gateway, key: str) -> None: + async def call() -> object: + async with ( + httpx.AsyncClient(timeout=15, trust_env=False) as transport, + anthropic.AsyncAnthropic( + api_key=key, base_url=_origin(gateway), max_retries=0, http_client=transport + ) as client, + ): + return await client.messages.create( + model=SHARED_MODEL, max_tokens=16, messages=[{"role": "user", "content": "sdk"}] + ) + + _run(call) + + +@pytest.mark.parametrize( + ("call", "error"), + ( + pytest.param(_openai_chat, openai.BadRequestError, id="openai-sync-chat"), + pytest.param(_async_openai_chat, openai.BadRequestError, id="openai-async-chat"), + pytest.param(_openai_responses, openai.BadRequestError, id="openai-sync-responses"), + pytest.param(_openai_embeddings, openai.BadRequestError, id="openai-sync-embeddings"), + pytest.param(_anthropic_messages, anthropic.BadRequestError, id="anthropic-sync-messages"), + pytest.param(_async_anthropic_messages, anthropic.BadRequestError, id="anthropic-async-messages"), + ), +) +def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string( + guarded: Gateway, peer: Wire, call: SdkCall, error: type[Exception] +) -> None: + with guarded.scenario() as scenario: + peer.drain() + + with pytest.raises(error, match="is the litellm_params.model of more than one model group"): + call(guarded, scenario.key()) + assert peer.drain() == () + + def test_a_model_group_name_still_routes_inside_its_own_group_when_the_check_is_on( guarded: Gateway, peer: Wire ) -> None: @@ -357,3 +498,71 @@ def test_the_admin_ui_config_update_turns_the_check_on_and_off_without_a_restart assert refused.status_code == 400, refused.text assert after.status_code == 200, after.text assert len(served) == 2 + + +@pytest.mark.parametrize( + ("value", "enabled"), + ( + pytest.param("TRUE", True, id="upper-case-true"), + pytest.param("yes", False, id="yes"), + pytest.param(1, False, id="integer-one"), + pytest.param("", False, id="empty-string"), + ), +) +def test_only_a_true_value_from_the_admin_ui_turns_the_check_on( + toggled: Gateway, peer: Wire, value: JsonValue, enabled: bool +) -> None: + with toggled.scenario() as scenario: + key: Final = scenario.key() + scenario.cleanups.callback( + toggled.post, "/config/update", {"router_settings": {"enable_cross_model_group_collision_check": False}} + ) + toggled.post("/config/update", {"router_settings": {"enable_cross_model_group_collision_check": value}}) + peer.drain() + response: Final = toggled.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) + served: Final = _served_by(peer.drain()) + + assert response.status_code == (400 if enabled else 200), response.text + assert len(served) == (0 if enabled else 1) + + +def _burst_call(gateway: Gateway, key: str, index: int) -> tuple[bool, int | None, str]: + raw: Final = index % 2 == 0 + build: Final = (_chat, _messages, _responses)[index // 2 % 3] + marker: Final = f"collision-burst-{index}-{uuid.uuid4().hex}" + path, body = build(SHARED_MODEL if raw else "collision-chain-b", marker) + try: + response: Final = gateway.request("POST", path, {**body, "stream": raw and index % 4 == 0}, key=key) + except httpx.HTTPError: + return raw, None, marker + return raw, response.status_code, marker + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 180) +def test_the_refusal_holds_on_both_workers_through_a_worker_kill_mid_burst(peer: Wire, tmp_path: Path) -> None: + path: Final = _config(peer, tmp_path, {"enable_cross_model_group_collision_check": True}) + with ( + gateway_from_environment() as base, + owned_proxy_process(base, tmp_path, {}, config=path, workers=2) as owned, + owned.gateway.scenario() as scenario, + ): + key: Final = scenario.key() + workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30) + peer.drain() + with ThreadPoolExecutor(max_workers=BURST) as pool: + futures: Final = [pool.submit(_burst_call, owned.gateway, key, index) for index in range(BURST)] + with owned.replacing_worker(min(workers)) as kill: + kill() + outcomes: Final = [future.result() for future in futures] + upstream_bodies: Final = tuple(request.body.decode() for request in peer.drain()) + probes: Final = tuple( + owned.gateway.request("POST", *_chat(SHARED_MODEL, uuid.uuid4().hex), key=key) for _ in range(8) + ) + control: Final = owned.gateway.request("POST", *_chat("collision-chain-b", uuid.uuid4().hex), key=key) + + assert all(status in (400, None) for raw, status, _ in outcomes if raw), outcomes + assert any(status == 400 for raw, status, _ in outcomes if raw), outcomes + assert all(status in (200, None) for raw, status, _ in outcomes if not raw), outcomes + assert not any(marker in body for raw, _, marker in outcomes if raw for body in upstream_bodies) + assert [probe.status_code for probe in probes] == [400] * 8, [probe.text for probe in probes] + assert control.status_code == 200, control.text diff --git a/tests/unit/test_router/test_router_get_deployments.py b/tests/unit/test_router/test_router_get_deployments.py index 9aaa037737d..f9f8c7a26d8 100644 --- a/tests/unit/test_router/test_router_get_deployments.py +++ b/tests/unit/test_router/test_router_get_deployments.py @@ -589,6 +589,45 @@ def test_collision_check_refuses_a_team_whose_model_shares_the_litellm_model_wit assert _pick_for_team(router, "team-2", specific_deployment) == "chain-b-1" +def _served_model_list() -> list[dict[str, object]]: + return [ + { + **deployment, + "litellm_params": { + **deployment["litellm_params"], + "mock_response": f"served by {deployment['model_info']['id']}", + }, + } + for deployment in _COLLIDING_MODEL_LIST + ] + + +@pytest.mark.parametrize( + "fallbacks", + ( + pytest.param([{"*": ["solo"]}], id="wildcard-fallback"), + pytest.param([{"alias": ["solo"]}, {_SHARED_LITELLM_MODEL: ["solo"]}], id="keyed-fallback"), + ), +) +@pytest.mark.parametrize("model", ("alias", _SHARED_LITELLM_MODEL)) +def test_collision_check_refusal_is_not_rescued_by_a_fallback( + fallbacks: list[dict[str, list[str]]], model: str +) -> None: + router: Final = Router( + model_list=_served_model_list(), + model_group_alias={"alias": _SHARED_LITELLM_MODEL}, + fallbacks=fallbacks, + enable_cross_model_group_collision_check=True, + ) + messages: Final = [{"role": "user", "content": "hi"}] + + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + router.completion(model=model, messages=messages) + with pytest.raises(litellm.BadRequestError, match=f"model={_SHARED_LITELLM_MODEL} is the litellm_params.model"): + asyncio.run(router.acompletion(model=model, messages=messages)) + assert router.completion(model="chain-b", messages=messages).choices[0].message.content == "served by chain-b-1" # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # completion returns an untyped response union + + @pytest.mark.parametrize( ("setting", "enabled"), ((True, True), ("true", True), (False, False), ("false", False)), From 4c2ff28f985bada891764125e8babf8a15d8c3f5 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:03:39 -0700 Subject: [PATCH 3/4] test(router): match the collision refusal without regex metacharacters --- tests/integration/routing/test_raw_model_group_collision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py index e9a12172739..27eaa286000 100644 --- a/tests/integration/routing/test_raw_model_group_collision.py +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -347,7 +347,7 @@ def test_sdk_clients_get_a_bad_request_error_for_a_shared_raw_model_string( with guarded.scenario() as scenario: peer.drain() - with pytest.raises(error, match="is the litellm_params.model of more than one model group"): + with pytest.raises(error, match="more than one model group"): call(guarded, scenario.key()) assert peer.drain() == () From 26efe879e80123a6071550d9ddb2b8c8d43b2bbc Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:35:13 -0700 Subject: [PATCH 4/4] fix(router): count a team's pre-public-name models as one group, keep Router args in place A team deployment created before team_public_model_name existed has only its internal model_name__ name, so two copies of one such team model counted as two model groups and the opt-in check refused them. A team whose matching deployments include one without a public name now counts as one group. The new Router argument moves to the end of the signature so no existing argument changes position. --- litellm/router.py | 4 +- litellm/router_utils/common_utils.py | 21 ++++-- .../routing/test_raw_model_group_collision.py | 71 ++++++++++++++++++- .../test_router_utils_common_utils.py | 38 +++++++++- .../test_router_get_deployments.py | 31 +++++++- 5 files changed, 153 insertions(+), 12 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 2fd89d3e68c..082c74c030d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1211,10 +1211,10 @@ class Router: background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, enable_mid_stream_fallback_continuation: bool = False, - enable_cross_model_group_collision_check: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, + enable_cross_model_group_collision_check: bool = False, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -1254,9 +1254,9 @@ class Router: ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. - enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). + enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. Returns: Router: An instance of the litellm.Router class. diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 585b0b9b633..e9eed6a9174 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -331,12 +331,17 @@ def filter_team_based_models( def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]: - """A team deployment's model group is its team and public name; any other deployment's is its `model_name`.""" + """A team deployment's model group is its team and public name, with no name for a team deployment created + before public names; any other deployment's is its `model_name`.""" model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {}) + team_id: Final = model_info.get("team_id") public_name: Final = model_info.get("team_public_model_name") if isinstance(public_name, str) and public_name: - return model_info.get("team_id"), public_name - return None, deployment["model_name"] + return team_id, public_name + model_name: Final = deployment["model_name"] + if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"): + return team_id, None + return None, model_name class RawModelGroupCollisionError(BadRequestError): @@ -344,8 +349,14 @@ class RawModelGroupCollisionError(BadRequestError): def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None: - """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.""" - if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2: + """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group. + + A team with a matching deployment created before public names counts as one model group. + """ + groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments) + unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None) + counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups) + if len(counted) < 2: return raise RawModelGroupCollisionError( message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. " diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py index 27eaa286000..c91acc41a82 100644 --- a/tests/integration/routing/test_raw_model_group_collision.py +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -36,6 +36,7 @@ DEPLOYMENTS: Final = ( CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"}) TEAM_MODEL: Final = "openai/integration-team-raw-model" TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"}) +TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names" ALIAS: Final = "collision-alias" POOLED_CALLS: Final = 48 JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -106,14 +107,26 @@ def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, } +def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]: + return { + "model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}", + "litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"}, + "model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES}, + } + + def _config( wire: Wire, directory: Path, router_settings: dict[str, JsonValue], extra_deployments: tuple[tuple[str, str, str], ...] = (), + team_deployments: tuple[dict[str, JsonValue], ...] = (), ) -> Path: config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) - config["model_list"] = [_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)] + config["model_list"] = [ + *(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)), + *team_deployments, + ] config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings} path: Final = directory / "raw-model-group-collision.yaml" path.write_text(yaml.safe_dump(config)) @@ -159,6 +172,24 @@ def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.Tem yield proxy +@pytest.fixture(scope="module") +def guarded_with_team_models_from_before_public_names( + peer: Wire, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names") + path: Final = _config( + peer, + directory, + {"enable_cross_model_group_collision_check": True}, + team_deployments=( + *(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS), + _team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL), + ), + ) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + @pytest.fixture(scope="module") def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled") @@ -475,6 +506,44 @@ def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_f assert received == ("collision-solo-1",) +def test_a_team_model_created_before_public_names_counts_as_one_model_group( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + _team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3") + key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + peer.drain() + responses: Final = tuple( + gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses] + assert len(served) == 12 + assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"} + + +def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + other_key: Final = scenario.key() + peer.drain() + refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key) + served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key) + received: Final = _served_by(peer.drain()) + + assert refused.status_code == 400, refused.text + assert SOLO_MODEL in refused.text + assert served.status_code == 200, served.text + assert received == ("collision-solo-1",) + + def _router_setting(gateway: Gateway, name: str) -> JsonValue: fields: Final = gateway.get("/router/settings")["fields"] assert isinstance(fields, list) diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index d8d07835295..3cf2cde1d20 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -968,8 +968,12 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"] -def _group_member(model_name: str, team: tuple[str, str] | None = None) -> DeploymentTypedDict: - model_info: Final[dict[str, str]] = {} if team is None else {"team_id": team[0], "team_public_model_name": team[1]} +def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict: + model_info: Final[dict[str, str]] = {} + if team is not None: + model_info["team_id"] = team[0] + if team[1] is not None: + model_info["team_public_model_name"] = team[1] return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info} @@ -1001,6 +1005,36 @@ def _group_member(model_name: str, team: tuple[str, str] | None = None) -> Deplo True, id="team-model-and-proxy-group", ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))), + False, + id="team-model-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))), + False, + id="team-deployments-from-before-and-after-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))), + True, + id="two-teams-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")), + True, + id="team-model-created-before-public-names-and-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("chain-a")), + True, + id="team-deployment-named-like-a-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")), + False, + id="team-deployment-in-a-proxy-group", + ), ), ) def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups( diff --git a/tests/unit/test_router/test_router_get_deployments.py b/tests/unit/test_router/test_router_get_deployments.py index f9f8c7a26d8..24fd5180f9b 100644 --- a/tests/unit/test_router/test_router_get_deployments.py +++ b/tests/unit/test_router/test_router_get_deployments.py @@ -528,11 +528,14 @@ def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_mod assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"} -def _team_deployment(deployment_id: str, team_id: str) -> dict[str, object]: +def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]: + model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id} + if public_name is not None: + model_info["team_public_model_name"] = public_name return { "model_name": f"model_name_{team_id}_{deployment_id}", "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, - "model_info": {"id": deployment_id, "team_id": team_id, "team_public_model_name": "team-model"}, + "model_info": model_info, } @@ -562,6 +565,30 @@ def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> s return asyncio.run(pick()) +@pytest.mark.parametrize("specific_deployment", (True, False)) +@pytest.mark.parametrize( + "public_names", + ( + pytest.param((None, None), id="both-created-before-public-names"), + pytest.param((None, "team-model"), id="one-created-before-public-names"), + ), +) +def test_collision_check_serves_a_team_model_created_before_public_names( + public_names: tuple[str | None, str | None], specific_deployment: bool +) -> None: + router: Final = Router( + model_list=[ + _team_deployment("team-1-a", "team-1", public_names[0]), + _team_deployment("team-1-b", "team-1", public_names[1]), + ], + enable_cross_model_group_collision_check=True, + ) + + picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)} + + assert picked == {"team-1-a", "team-1-b"} + + @pytest.mark.parametrize("specific_deployment", (True, False)) def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model( specific_deployment: bool,