fix(router): count a team's pre-public-name models as one group, keep Router args in place

A team deployment created before team_public_model_name existed has only its
internal model_name_<team>_<uuid> name, so two copies of one such team model
counted as two model groups and the opt-in check refused them. A team whose
matching deployments include one without a public name now counts as one
group. The new Router argument moves to the end of the signature so no
existing argument changes position.
This commit is contained in:
yucheng-berri 2026-10-20 17:35:13 -07:00
parent 4c2ff28f98
commit 26efe879e8
5 changed files with 153 additions and 12 deletions

View file

@ -1211,10 +1211,10 @@ class Router:
background_health_check_model_groups: Sequence[str] | None = None,
enable_weighted_failover: bool = False,
enable_mid_stream_fallback_continuation: bool = False,
enable_cross_model_group_collision_check: bool = False,
fallback_access_check: FallbackAccessCheck | None = None,
fallback_budget_check: FallbackBudgetCheck | None = None,
auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
enable_cross_model_group_collision_check: bool = False,
) -> None:
"""
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
@ -1254,9 +1254,9 @@ class Router:
ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False.
enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False.
fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
enable_cross_model_group_collision_check (bool): When True, a request whose model matches no model group but is the `litellm_params.model` of deployments in more than one model group is refused with a 400 instead of being load balanced across all of those groups. Defaults to False.
Returns:
Router: An instance of the litellm.Router class.

View file

@ -331,12 +331,17 @@ def filter_team_based_models(
def _model_group_of(deployment: Mapping[str, object]) -> tuple[object, object]:
"""A team deployment's model group is its team and public name; any other deployment's is its `model_name`."""
"""A team deployment's model group is its team and public name, with no name for a team deployment created
before public names; any other deployment's is its `model_name`."""
model_info: Final = _OBJECT_MAPPING_ADAPTER.validate_python(deployment.get("model_info") or {})
team_id: Final = model_info.get("team_id")
public_name: Final = model_info.get("team_public_model_name")
if isinstance(public_name, str) and public_name:
return model_info.get("team_id"), public_name
return None, deployment["model_name"]
return team_id, public_name
model_name: Final = deployment["model_name"]
if isinstance(team_id, str) and isinstance(model_name, str) and model_name.startswith(f"model_name_{team_id}_"):
return team_id, None
return None, model_name
class RawModelGroupCollisionError(BadRequestError):
@ -344,8 +349,14 @@ class RawModelGroupCollisionError(BadRequestError):
def refuse_litellm_model_spanning_model_groups(model: str, deployments: Sequence[Mapping[str, object]]) -> None:
"""Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group."""
if len(frozenset(_model_group_of(deployment) for deployment in deployments)) < 2:
"""Raise a 400 when the deployments matched by a raw `litellm_params.model` belong to more than one model group.
A team with a matching deployment created before public names counts as one model group.
"""
groups: Final = frozenset(_model_group_of(deployment) for deployment in deployments)
unnamed_teams: Final = frozenset(team for team, name in groups if team is not None and name is None)
counted: Final = frozenset((team, None) if team in unnamed_teams else (team, name) for team, name in groups)
if len(counted) < 2:
return
raise RawModelGroupCollisionError(
message=f"LiteLLM Router: model={model} is the litellm_params.model of more than one model group. "

View file

@ -36,6 +36,7 @@ DEPLOYMENTS: Final = (
CHAIN_A: Final = frozenset({"collision-a-1", "collision-a-2"})
TEAM_MODEL: Final = "openai/integration-team-raw-model"
TEAM_DEPLOYMENTS: Final = frozenset({"collision-team-1", "collision-team-2"})
TEAM_BEFORE_PUBLIC_NAMES: Final = "collision-team-before-public-names"
ALIAS: Final = "collision-alias"
POOLED_CALLS: Final = 48
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
@ -106,14 +107,26 @@ def _deployment(wire: Wire, group: str, identity: str, model: str) -> dict[str,
}
def _team_deployment_from_before_public_names(wire: Wire, identity: str, model: str) -> dict[str, JsonValue]:
return {
"model_name": f"model_name_{TEAM_BEFORE_PUBLIC_NAMES}_{identity}",
"litellm_params": {"model": model, "api_base": f"{wire.url}/{identity}/v1", "api_key": "synthetic-key"},
"model_info": {"id": identity, "team_id": TEAM_BEFORE_PUBLIC_NAMES},
}
def _config(
wire: Wire,
directory: Path,
router_settings: dict[str, JsonValue],
extra_deployments: tuple[tuple[str, str, str], ...] = (),
team_deployments: tuple[dict[str, JsonValue], ...] = (),
) -> Path:
config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
config["model_list"] = [_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)]
config["model_list"] = [
*(_deployment(wire, *deployment) for deployment in (*DEPLOYMENTS, *extra_deployments)),
*team_deployments,
]
config["router_settings"] = {**JSON_OBJECT.validate_python(config["router_settings"]), **router_settings}
path: Final = directory / "raw-model-group-collision.yaml"
path.write_text(yaml.safe_dump(config))
@ -159,6 +172,24 @@ def guarded_without_specific_deployment(peer: Wire, tmp_path_factory: pytest.Tem
yield proxy
@pytest.fixture(scope="module")
def guarded_with_team_models_from_before_public_names(
peer: Wire, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-before-public-names")
path: Final = _config(
peer,
directory,
{"enable_cross_model_group_collision_check": True},
team_deployments=(
*(_team_deployment_from_before_public_names(peer, identity, TEAM_MODEL) for identity in TEAM_DEPLOYMENTS),
_team_deployment_from_before_public_names(peer, "collision-team-solo-1", SOLO_MODEL),
),
)
with gateway_from_environment() as base, owned_proxy(base, directory, {}, config=path) as proxy:
yield proxy
@pytest.fixture(scope="module")
def toggled(peer: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
directory: Final = tmp_path_factory.mktemp("raw-model-collision-toggled")
@ -475,6 +506,44 @@ def test_a_team_model_on_a_raw_model_string_a_proxy_group_also_uses_is_refused_f
assert received == ("collision-solo-1",)
def test_a_team_model_created_before_public_names_counts_as_one_model_group(
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
) -> None:
gateway: Final = guarded_with_team_models_from_before_public_names
with gateway.scenario() as scenario:
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
_team_deployment(scenario, peer, TEAM_BEFORE_PUBLIC_NAMES, "collision-team-model", "collision-team-3")
key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
peer.drain()
responses: Final = tuple(
gateway.request("POST", *_chat(TEAM_MODEL, uuid.uuid4().hex), key=key) for _ in range(12)
)
served: Final = _served_by(peer.drain())
assert [response.status_code for response in responses] == [200] * 12, [r.text for r in responses]
assert len(served) == 12
assert set(served) <= {*TEAM_DEPLOYMENTS, "collision-team-3"}
def test_a_team_model_created_before_public_names_on_a_proxy_groups_raw_model_string_is_refused(
guarded_with_team_models_from_before_public_names: Gateway, peer: Wire
) -> None:
gateway: Final = guarded_with_team_models_from_before_public_names
with gateway.scenario() as scenario:
scenario.team(team_id=TEAM_BEFORE_PUBLIC_NAMES)
team_key: Final = scenario.key(team_id=TEAM_BEFORE_PUBLIC_NAMES)
other_key: Final = scenario.key()
peer.drain()
refused: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=team_key)
served: Final = gateway.request("POST", *_chat(SOLO_MODEL, uuid.uuid4().hex), key=other_key)
received: Final = _served_by(peer.drain())
assert refused.status_code == 400, refused.text
assert SOLO_MODEL in refused.text
assert served.status_code == 200, served.text
assert received == ("collision-solo-1",)
def _router_setting(gateway: Gateway, name: str) -> JsonValue:
fields: Final = gateway.get("/router/settings")["fields"]
assert isinstance(fields, list)

View file

@ -968,8 +968,12 @@ def test_router_warns_once_and_serves_no_flag_for_a_deployment_synced_from_the_d
assert "supports_max_reasoning_effort" not in router.get_model_list()[0]["litellm_params"]
def _group_member(model_name: str, team: tuple[str, str] | None = None) -> DeploymentTypedDict:
model_info: Final[dict[str, str]] = {} if team is None else {"team_id": team[0], "team_public_model_name": team[1]}
def _group_member(model_name: str, team: tuple[str, str | None] | None = None) -> DeploymentTypedDict:
model_info: Final[dict[str, str]] = {}
if team is not None:
model_info["team_id"] = team[0]
if team[1] is not None:
model_info["team_public_model_name"] = team[1]
return {"model_name": model_name, "litellm_params": {"model": "openai/shared-backend"}, "model_info": model_info}
@ -1001,6 +1005,36 @@ def _group_member(model_name: str, team: tuple[str, str] | None = None) -> Deplo
True,
id="team-model-and-proxy-group",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", None))),
False,
id="team-model-created-before-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t1_b", ("t1", "gpt-team"))),
False,
id="team-deployments-from-before-and-after-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("model_name_t2_a", ("t2", None))),
True,
id="two-teams-created-before-public-names",
),
pytest.param(
(_group_member("model_name_t1_a", ("t1", None)), _group_member("chain-a")),
True,
id="team-model-created-before-public-names-and-proxy-group",
),
pytest.param(
(_group_member("gpt-team", ("t1", None)), _group_member("chain-a")),
True,
id="team-deployment-named-like-a-proxy-group",
),
pytest.param(
(_group_member("gpt-team", ("t1", None)), _group_member("gpt-team")),
False,
id="team-deployment-in-a-proxy-group",
),
),
)
def test_refuse_litellm_model_spanning_model_groups_decides_on_distinct_model_groups(

View file

@ -528,11 +528,14 @@ def test_collision_check_applies_to_an_alias_that_points_at_a_shared_litellm_mod
assert {_pick_sync(router, "alias", False) for _ in range(60)} == {"chain-a-1", "chain-a-2", "chain-b-1"}
def _team_deployment(deployment_id: str, team_id: str) -> dict[str, object]:
def _team_deployment(deployment_id: str, team_id: str, public_name: str | None = "team-model") -> dict[str, object]:
model_info: Final[dict[str, str]] = {"id": deployment_id, "team_id": team_id}
if public_name is not None:
model_info["team_public_model_name"] = public_name
return {
"model_name": f"model_name_{team_id}_{deployment_id}",
"litellm_params": {"model": _SHARED_LITELLM_MODEL, "api_key": "test-key"},
"model_info": {"id": deployment_id, "team_id": team_id, "team_public_model_name": "team-model"},
"model_info": model_info,
}
@ -562,6 +565,30 @@ def _pick_for_team(router: Router, team_id: str, specific_deployment: bool) -> s
return asyncio.run(pick())
@pytest.mark.parametrize("specific_deployment", (True, False))
@pytest.mark.parametrize(
"public_names",
(
pytest.param((None, None), id="both-created-before-public-names"),
pytest.param((None, "team-model"), id="one-created-before-public-names"),
),
)
def test_collision_check_serves_a_team_model_created_before_public_names(
public_names: tuple[str | None, str | None], specific_deployment: bool
) -> None:
router: Final = Router(
model_list=[
_team_deployment("team-1-a", "team-1", public_names[0]),
_team_deployment("team-1-b", "team-1", public_names[1]),
],
enable_cross_model_group_collision_check=True,
)
picked: Final = {_pick_for_team(router, "team-1", specific_deployment) for _ in range(40)}
assert picked == {"team-1-a", "team-1-b"}
@pytest.mark.parametrize("specific_deployment", (True, False))
def test_collision_check_serves_each_team_its_own_model_when_another_team_shares_the_litellm_model(
specific_deployment: bool,