From 26efe879e80123a6071550d9ddb2b8c8d43b2bbc Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:35:13 -0700 Subject: [PATCH] fix(router): count a team's pre-public-name models as one group, keep Router args in place A team deployment created before team_public_model_name existed has only its internal model_name__ name, so two copies of one such team model counted as two model groups and the opt-in check refused them. A team whose matching deployments include one without a public name now counts as one group. The new Router argument moves to the end of the signature so no existing argument changes position. --- litellm/router.py | 4 +- litellm/router_utils/common_utils.py | 21 ++++-- .../routing/test_raw_model_group_collision.py | 71 ++++++++++++++++++- .../test_router_utils_common_utils.py | 38 +++++++++- .../test_router_get_deployments.py | 31 +++++++- 5 files changed, 153 insertions(+), 12 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 2fd89d3e68c..082c74c030d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1211,10 +1211,10 @@ class Router: background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, enable_mid_stream_fallback_continuation: bool = False, - enable_cross_model_group_collision_check: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, + enable_cross_model_group_collision_check: bool = False, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -1254,9 +1254,9 @@ class Router: ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. - enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). + enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False. Returns: Router: An instance of the litellm.Router class. diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 585b0b9b633..e9eed6a9174 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -331,12 +331,17 @@ def filter_team_based_models( def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]: - """A team deployment's model group is its team and public name; any other deployment's is its `model_name`.""" + """A team deployment's model group is its team and public name, with no name for a team deployment created + before public names; any other deployment's is its `model_name`.""" model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {}) + team_id: Final = model_info.get("team_id") public_name: Final = model_info.get("team_public_model_name") if isinstance(public_name, str) and public_name: - return model_info.get("team_id"), public_name - return None, deployment["model_name"] + return team_id, public_name + model_name: Final = deployment["model_name"] + if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"): + return team_id, None + return None, model_name class RawModelGroupCollisionError(BadRequestError): @@ -344,8 +349,14 @@ class RawModelGroupCollisionError(BadRequestError): def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None: - """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.""" - if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2: + """Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group. + + A team with a matching deployment created before public names counts as one model group. + """ + groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments) + unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None) + counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups) + if len(counted) < 2: return raise RawModelGroupCollisionError( message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. " diff --git a/tests/integration/routing/test_raw_model_group_collision.py b/tests/integration/routing/test_raw_model_group_collision.py index 27eaa286000..c91acc41a82 100644 --- a/tests/integration/routing/test_raw_model_group_collision.py +++ b/tests/integration/routing/test_raw_model_group_collision.py @@ -36,6 +36,7 @@ DEPLOYMENTS: Final = ( CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"}) TEAM_MODEL: Final = "openai/integration-team-raw-model" TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"}) +TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names" ALIAS: Final = "collision-alias" POOLED_CALLS: Final = 48 JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -106,14 +107,26 @@ def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str, } +def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]: + return { + "model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}", + "litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"}, + "model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES}, + } + + def _config( wire: Wire, directory: Path, router_settings: dict[str, JsonValue], extra_deployments: tuple[tuple[str, str, str], ...] = (), + team_deployments: tuple[dict[str, JsonValue], ...] = (), ) -> Path: config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) - config["model_list"] = [_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)] + config["model_list"] = [ + *(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)), + *team_deployments, + ] config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings} path: Final = directory / "raw-model-group-collision.yaml" path.write_text(yaml.safe_dump(config)) @@ -159,6 +172,24 @@ def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.Tem yield proxy +@pytest.fixture(scope="module") +def guarded_with_team_models_from_before_public_names( + peer: Wire, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names") + path: Final = _config( + peer, + directory, + {"enable_cross_model_group_collision_check": True}, + team_deployments=( + *(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS), + _team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL), + ), + ) + with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy: + yield proxy + + @pytest.fixture(scope="module") def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled") @@ -475,6 +506,44 @@ def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_f assert received == ("collision-solo-1",) +def test_a_team_model_created_before_public_names_counts_as_one_model_group( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + _team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3") + key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + peer.drain() + responses: Final = tuple( + gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12) + ) + served: Final = _served_by(peer.drain()) + + assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses] + assert len(served) == 12 + assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"} + + +def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused( + guarded_with_team_models_from_before_public_names: Gateway, peer: Wire +) -> None: + gateway: Final = guarded_with_team_models_from_before_public_names + with gateway.scenario() as scenario: + scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES) + team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES) + other_key: Final = scenario.key() + peer.drain() + refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key) + served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key) + received: Final = _served_by(peer.drain()) + + assert refused.status_code == 400, refused.text + assert SOLO_MODEL in refused.text + assert served.status_code == 200, served.text + assert received == ("collision-solo-1",) + + def _router_setting(gateway: Gateway, name: str) -> JsonValue: fields: Final = gateway.get("/router/settings")["fields"] assert isinstance(fields, list) diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index d8d07835295..3cf2cde1d20 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -968,8 +968,12 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"] -def _group_member(model_name: str, team: tuple[str, str] | None = None) -> DeploymentTypedDict: - model_info: Final[dict[str, str]] = {} if team is None else {"team_id": team[0], "team_public_model_name": team[1]} +def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict: + model_info: Final[dict[str, str]] = {} + if team is not None: + model_info["team_id"] = team[0] + if team[1] is not None: + model_info["team_public_model_name"] = team[1] return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info} @@ -1001,6 +1005,36 @@ def _group_member(model_name: str, team: tuple[str, str] | None = None) -> Deplo True, id="team-model-and-proxy-group", ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))), + False, + id="team-model-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))), + False, + id="team-deployments-from-before-and-after-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))), + True, + id="two-teams-created-before-public-names", + ), + pytest.param( + (_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")), + True, + id="team-model-created-before-public-names-and-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("chain-a")), + True, + id="team-deployment-named-like-a-proxy-group", + ), + pytest.param( + (_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")), + False, + id="team-deployment-in-a-proxy-group", + ), ), ) def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups( diff --git a/tests/unit/test_router/test_router_get_deployments.py b/tests/unit/test_router/test_router_get_deployments.py index f9f8c7a26d8..24fd5180f9b 100644 --- a/tests/unit/test_router/test_router_get_deployments.py +++ b/tests/unit/test_router/test_router_get_deployments.py @@ -528,11 +528,14 @@ def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_mod assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"} -def _team_deployment(deployment_id: str, team_id: str) -> dict[str, object]: +def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]: + model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id} + if public_name is not None: + model_info["team_public_model_name"] = public_name return { "model_name": f"model_name_{team_id}_{deployment_id}", "litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"}, - "model_info": {"id": deployment_id, "team_id": team_id, "team_public_model_name": "team-model"}, + "model_info": model_info, } @@ -562,6 +565,30 @@ def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> s return asyncio.run(pick()) +@pytest.mark.parametrize("specific_deployment", (True, False)) +@pytest.mark.parametrize( + "public_names", + ( + pytest.param((None, None), id="both-created-before-public-names"), + pytest.param((None, "team-model"), id="one-created-before-public-names"), + ), +) +def test_collision_check_serves_a_team_model_created_before_public_names( + public_names: tuple[str | None, str | None], specific_deployment: bool +) -> None: + router: Final = Router( + model_list=[ + _team_deployment("team-1-a", "team-1", public_names[0]), + _team_deployment("team-1-b", "team-1", public_names[1]), + ], + enable_cross_model_group_collision_check=True, + ) + + picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)} + + assert picked == {"team-1-a", "team-1-b"} + + @pytest.mark.parametrize("specific_deployment", (True, False)) def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model( specific_deployment: bool,