diff --git a/litellm/router.py b/litellm/router.py index 7d566bd0744..f86dd859b4c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12677,7 +12677,11 @@ class Router: return "", "" def _set_model_group_info( - self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None + self, + model_group: str, + user_facing_model_group_name: str, + team_id: str | None = None, + deployment: Deployment | None = None, ) -> ModelGroupInfo | None: """ For a given model group name, return the combined model info @@ -12695,7 +12699,11 @@ class Router: configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None reasoning_efforts_initialized = False reasoning_efforts_unknown = False - model_list: Final = self.get_model_list_of_routed_group(model_group) + model_list: Final = ( + (cast(DeploymentTypedDict, deployment.model_dump(exclude_none=True)),) # cast-ok: router deployment schema + if deployment is not None + else self.get_model_list_of_routed_group(model_group) + ) model_group_description: Final[str | None] = next( (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, @@ -12703,7 +12711,8 @@ class Router: for model in model_list: is_match = False if ( - "model_name" in model + deployment is not None + or "model_name" in model and model["model_name"] == model_group or "model_name" in model and self.pattern_router.route(model_group) is not None @@ -12964,8 +12973,9 @@ class Router: - ModelGroupInfo if able to construct a model group - None if error constructing model group info or hidden model group (unless include_hidden) """ + deployment: Final = self._pinned_deployment(model_group) ## Check if model group alias - if model_group in self.model_group_alias: + if deployment is None and model_group in self.model_group_alias: item: Final = self.model_group_alias[model_group] if isinstance(item, str): _router_model_group = item @@ -12985,7 +12995,10 @@ class Router: ## Check if actual model return self._set_model_group_info( - model_group=model_group, user_facing_model_group_name=model_group, team_id=team_id + model_group=model_group, + user_facing_model_group_name=model_group, + team_id=team_id, + deployment=deployment, ) async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 62510dac26d..05c2e1e9284 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2415,6 +2415,67 @@ def test_cached_get_model_group_info(): assert result5 is result6 +@pytest.mark.parametrize("wildcard", [False, True]) +@pytest.mark.parametrize("collision", ["none", "group", "alias", "hidden-alias"]) +def test_model_group_info_respects_deployment_id_precedence( + wildcard: bool, collision: str, caplog: pytest.LogCaptureFixture +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "example-pool", + "litellm_params": {"model": "openai/example-model", "api_key": "fake", "rpm": 10, "tpm": 100}, + "model_info": {"id": "deployment-a", "input_cost_per_token": 0.001}, + }, + { + "model_name": "deployment-a" if collision == "group" else "example-pool", + "litellm_params": {"model": "openai/example-model", "api_key": "fake", "rpm": 20, "tpm": 200}, + "model_info": {"id": "deployment-b", "input_cost_per_token": 0.002}, + }, + *([{"model_name": "*", "litellm_params": {"model": "*"}}] if wildcard else []), + ], + model_group_alias={ + **({"deployment-a": "example-pool"} if collision == "alias" else {}), + **({"deployment-a": {"model": "example-pool", "hidden": True}} if collision == "hidden-alias" else {}), + "visible": "example-pool", + }, + ) + before: Final = copy.deepcopy(router.model_list) + with caplog.at_level(logging.ERROR, logger="LiteLLM Router"): + info: Final = router.get_model_group_info("deployment-a") + assert info is not None + assert (info.model_group, info.providers) == ("deployment-a", ["openai"]) + assert (info.rpm, info.tpm, info.input_cost_per_token) == ( + (20, 200, 0.002) if collision == "group" else (10, 100, 0.001) + ) + assert all(record.levelno < logging.ERROR for record in caplog.records) + assert router.model_list == before + pooled: Final = router.get_model_group_info("visible") + assert pooled is not None + assert (pooled.rpm, pooled.tpm) == ((10, 100) if collision == "group" else (30, 300)) + + +def test_model_group_info_does_not_resolve_an_alias_target_as_a_deployment_id() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "example-pool", + "litellm_params": {"model": "openai/example-model", "api_key": "fake", "rpm": 10}, + "model_info": {"id": "deployment-a"}, + }, + { + "model_name": "deployment-a", + "litellm_params": {"model": "openai/example-model", "api_key": "fake", "rpm": 20}, + "model_info": {"id": "deployment-b"}, + }, + ], + model_group_alias={"visible": "deployment-a"}, + ) + info: Final = router.get_model_group_info("visible") + assert info is not None + assert (info.model_group, info.providers, info.rpm) == ("visible", ["openai"], 20) + + def test_model_group_info_cost_from_db_model_info(): """ When get_deployment_model_info fails (model_info is None fallback),