diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..b6fbd67aaec 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -664,23 +664,10 @@ def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[Deploy """ The deployments a request to ``model`` routes to, for the gates that waive budget checks. - The router resolves a ``model_group_alias`` name to its target exactly once and serves that - group, never a real deployment that shares the alias's name. ``Router.get_model_list()`` on the - routed group reads those deployments unless the group is itself an alias key, where it would - follow the alias a second hop the router never takes; then the deployments named after the - group count, or the wildcard route serving the group when none carries its name, the same - fallback ``get_model_list()`` takes. + Resolve the complete alias chain before looking up the served deployments. """ routed_group: Final = llm_router.routable_model_group(model) - if llm_router.routable_model_group(routed_group) == routed_group: - return llm_router.get_model_list(model_name=routed_group) or () - named: Final = llm_router.get_model_list_from_model_alias(model_name=model) - if named: - return named - return tuple( - DeploymentTypedDict(**deployment) - for deployment in llm_router.pattern_router.get_deployments_by_pattern(model=routed_group) - ) + return llm_router.get_model_list(model_name=routed_group) or () def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool: diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..db7a50563a3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12740,14 +12740,10 @@ class Router: ## Check if model group alias if model_group in self.model_group_alias: item: Final = self.model_group_alias[model_group] - if isinstance(item, str): - _router_model_group = item - elif isinstance(item, dict): - if item["hidden"] is True and not include_hidden: - return None - else: - _router_model_group = item["model"] - else: + if isinstance(item, dict) and item.get("hidden") is True and not include_hidden: + return None + _router_model_group: Final = self.get_model_from_alias(model_group) + if _router_model_group is None: return None return self._set_model_group_info( @@ -13560,14 +13556,10 @@ class Router: alias_items = list(self.model_group_alias.items()) for model_alias, model_value in alias_items: - if isinstance(model_value, str): - _router_model_name: str = model_value - elif isinstance(model_value, dict): - _model_value = RouterModelGroupAliasItem(**model_value) - if _model_value["hidden"] is True and model_name is None: - continue - _router_model_name = _model_value["model"] - else: + if isinstance(model_value, dict) and model_value.get("hidden") is True and model_name is None: + continue + _router_model_name = self.get_model_from_alias(model_alias) + if _router_model_name is None: continue if (alias_group := self.get_routing_group(_router_model_name)) is not None: @@ -13831,9 +13823,7 @@ class Router: the wildcard route matching it when neither exists. Unlike get_model_list, model_group's own model_group_alias entry is not - followed. The router resolves an alias exactly once, so a group reached as - an alias target is served by its own deployments, never by a second hop: - in the chain X -> T -> U a request to X is served from T's deployments. + followed. The caller has already resolved the complete alias chain. """ named: Final = [ *self._get_all_deployments(model_name=model_group), diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index a00d84babf3..ffb40dc2fbe 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -103,15 +103,19 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No entries: the map can come from a key or team row rather than from validated config, so a bad value must not raise mid-request. - Returns the target model group, or None when the map does not rewrite ``model``. + Returns the final target model group, or None when no alias applies or the chain cycles. """ - if not isinstance(model_group_alias, Mapping): + if not isinstance(model_group_alias, Mapping) or model not in model_group_alias: return None - entry: Final = model_group_alias.get(model) - target: Final = entry.get("model") if isinstance(entry, Mapping) else entry - if not isinstance(target, str) or not target: - return None - return target + aliases: Final = _OBJECT_MAPPING_ADAPTER.validate_python(model_group_alias) + current: str = model # rebind-ok: iterative traversal supports chains beyond Python's recursion limit + for _ in range(len(aliases) + 1): + entry = aliases.get(current) + target = _OBJECT_MAPPING_ADAPTER.validate_python(entry).get("model") if isinstance(entry, Mapping) else entry + if not isinstance(target, str) or not target: + return current if current != model else None + current = target + return None def resolve_served_model( diff --git a/tests/integration/authorization/test_alias_chain_budget_waiver.py b/tests/integration/authorization/test_alias_chain_budget_waiver.py index 0908fa0691c..03b1bdfd44b 100644 --- a/tests/integration/authorization/test_alias_chain_budget_waiver.py +++ b/tests/integration/authorization/test_alias_chain_budget_waiver.py @@ -293,7 +293,7 @@ def _await_rig(rig: ChainRig) -> None: tail_id: Final = _deployment_id(rig.tail_free) for candidate in rig.candidates: _await_served_by(candidate, rig.mid_free, paid_id) - _await_served_by(candidate, rig.failing_free, paid_id) + _await_served_by(candidate, rig.failing_free, _deployment_id(rig.failing_free)) _await_served_by(candidate, rig.mid_paid, tail_id) _await_served_by(candidate, rig.unpriced, tail_id) @@ -331,11 +331,14 @@ def _chain_rig() -> Generator[ChainRig]: ghost_entry=f"chain-ghost-entry-{suffix}", failing_entry=f"chain-failing-entry-{suffix}", ) + free_middle: Final = f"{rig.entry}-middle" + failing_middle: Final = f"{rig.failing_entry}-middle" aliases: Final[Mapping[str, JsonValue]] = { - rig.entry: rig.mid_free, - rig.hidden_entry: hidden(rig.mid_free), - rig.shown_entry: {"model": rig.mid_free, "hidden": False}, - rig.null_entry: {"model": rig.mid_free, "hidden": None}, + rig.entry: free_middle, + rig.hidden_entry: hidden(free_middle), + rig.shown_entry: {"model": free_middle, "hidden": False}, + rig.null_entry: {"model": free_middle, "hidden": None}, + free_middle: rig.tail_free, rig.mid_free: rig.paid, rig.reverse_entry: rig.mid_paid, rig.mid_paid: rig.tail_free, @@ -343,8 +346,8 @@ def _chain_rig() -> Generator[ChainRig]: rig.unpriced: rig.tail_free, rig.ghost_entry: rig.ghost_mid, rig.ghost_mid: rig.paid, - rig.failing_entry: rig.failing_free, - rig.failing_free: rig.paid, + rig.failing_entry: failing_middle, + failing_middle: rig.failing_free, } install_aliases(gateway, aliases) scenario.cleanups.callback(remove_aliases, gateway, frozenset(aliases)) @@ -524,46 +527,47 @@ def test_chain_middle_by_name_is_judged_by_its_own_alias(rig: ChainRig) -> None: assert row["model_group"] == rig.mid_free, row -def test_reverse_chain_is_priced_by_the_middle_it_is_served_from(rig: ChainRig) -> None: +def test_chain_through_a_paid_middle_is_priced_by_its_free_target(rig: ChainRig) -> None: free_marker: Final = "reverse-free-" + uuid.uuid4().hex - billed_marker: Final = "reverse-billed-" + uuid.uuid4().hex + entry_marker: Final = "reverse-entry-" + uuid.uuid4().hex with rig.gateway.scenario() as scenario: exhausted: Final = _exhausted_key(rig, scenario) for candidate in rig.candidates: - _assert_refused_on_every_route(rig, candidate, exhausted, rig.reverse_entry) + _assert_raw_chat_served(rig, candidate, rig.reverse_entry, exhausted, "reverse-chain") served: Final = fresh_chat(rig.gateway, rig.mid_paid, exhausted, free_marker) assert served.status_code == 200, served.text assert served.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict(served.headers) assert_free_row(landed_once(exhausted, free_marker), rig.mid_paid) - budgeted: Final = scenario.key(max_budget=HEADROOM) - billed: Final = fresh_chat(rig.gateway, rig.reverse_entry, budgeted, billed_marker) - assert billed.status_code == 200, billed.text - assert billed.headers["x-litellm-model-id"] == _deployment_id(rig.mid_paid), dict(billed.headers) - row: Final = landed_once(budgeted, billed_marker) - assert math.isclose(float(str(row["spend"])), PAID_REPLY_COST), row - assert row["model_group"] == rig.reverse_entry, row + entry_response: Final = fresh_chat(rig.gateway, rig.reverse_entry, exhausted, entry_marker) + assert entry_response.status_code == 200, entry_response.text + assert entry_response.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict( + entry_response.headers + ) + assert_free_row(landed_once(exhausted, entry_marker), rig.reverse_entry) -def test_chain_through_a_cost_map_priced_middle_stays_budgeted(rig: ChainRig) -> None: +def test_chain_through_a_cost_map_priced_middle_uses_the_free_targets_budget_waiver(rig: ChainRig) -> None: with rig.gateway.scenario() as scenario: key: Final = _exhausted_key(rig, scenario) for candidate in rig.candidates: - _assert_refused_on_every_route(rig, candidate, key, rig.unpriced_entry) + _assert_raw_chat_served(rig, candidate, rig.unpriced_entry, key, "unpriced-middle") def _assert_ghost_chain(rig: ChainRig, candidate: Gateway, key: str) -> None: - assert _group_rows(candidate, rig.ghost_entry) == () - assert rig.ghost_entry not in _listed_input_prices(candidate) + _assert_price( + _price_of(candidate, rig.ghost_entry), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",) + ) + assert _listed_input_prices(candidate)[rig.ghost_entry] == PAID_INPUT_COST_PER_TOKEN refused: Final = fresh_chat(candidate, rig.ghost_entry, key, "ghost-exhausted-" + uuid.uuid4().hex) assert refused.status_code == BUDGET_EXCEEDED, refused.text - unroutable: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex) - assert unroutable.status_code == 400, unroutable.text - assert "no healthy deployments" in unroutable.text, unroutable.text + served: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex) + assert served.status_code == 200, served.text + assert served.headers["x-litellm-model-id"] == _deployment_id(rig.paid), dict(served.headers) for path in ("/health/liveliness", "/model/info", "/v1/models"): assert candidate.request("GET", path).status_code == 200, path -def test_ghost_chain_reports_no_group_and_stays_unroutable(rig: ChainRig) -> None: +def test_chain_through_an_alias_without_deployments_routes_to_its_paid_target(rig: ChainRig) -> None: with rig.gateway.scenario() as scenario: key: Final = _exhausted_key(rig, scenario) for candidate in rig.candidates: @@ -590,7 +594,7 @@ def test_provider_failure_behind_a_chain_entry_reaches_the_caller(rig: ChainRig) _assert_no_served_row(key, failed_marker) -def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainRig) -> None: +def test_chain_to_a_paid_target_does_not_inherit_a_free_wildcards_budget_waiver(rig: ChainRig) -> None: prefix: Final = "wc" + uuid.uuid4().hex[:8] middle: Final = f"{prefix}/gpt-4o-mini" other: Final = f"{prefix}/gpt-4o" @@ -601,7 +605,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle})) paid_id: Final = _deployment_id(rig.paid) for candidate in rig.candidates: - _await_served_by(candidate, entry, route) + _await_served_by(candidate, entry, paid_id) _await_served_by(candidate, middle, paid_id) key: Final = _exhausted_key(rig, scenario) for candidate in rig.candidates: @@ -610,7 +614,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR _assert_price( _price_of(candidate, middle), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",) ) - _assert_served_by_route(candidate, entry, key, route, "wildcard-served") + _assert_refused_on_every_route(rig, candidate, key, entry) _assert_refused(rig, candidate, key, middle, "wildcard-middle") @@ -622,21 +626,21 @@ def _assert_served_by_route(candidate: Gateway, model: str, key: str, route: str assert_free_row(landed_once(key, marker), model) -def test_chain_through_a_priced_wildcard_route_stays_budgeted(rig: ChainRig) -> None: +def test_chain_to_a_free_target_takes_precedence_over_a_priced_wildcard_route(rig: ChainRig) -> None: prefix: Final = "pwc" + uuid.uuid4().hex[:8] middle: Final = f"{prefix}/gpt-4o-mini" entry: Final = "priced-wildcard-entry-" + uuid.uuid4().hex with rig.gateway.scenario() as scenario: - route: Final = _wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN) + _wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN) install_aliases(rig.gateway, {entry: middle, middle: rig.tail_free}) scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle})) tail_id: Final = _deployment_id(rig.tail_free) for candidate in rig.candidates: - _await_served_by(candidate, entry, route) + _await_served_by(candidate, entry, tail_id) _await_served_by(candidate, middle, tail_id) key: Final = _exhausted_key(rig, scenario) for candidate in rig.candidates: - _assert_refused_on_every_route(rig, candidate, key, entry) + _assert_raw_chat_served(rig, candidate, entry, key, "priced-wildcard-free-target") def _assert_hostile_group_queries_answered(rig: ChainRig, candidate: Gateway) -> None: @@ -738,7 +742,7 @@ def test_duplicate_model_field_is_judged_by_its_last_value(rig: ChainRig) -> Non def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> None: marker: Final = "restricted-served-" + uuid.uuid4().hex with rig.gateway.scenario() as scenario: - outsider: Final = scenario.key(models=[rig.tail_free, rig.paid], max_budget=BUDGET) + outsider: Final = scenario.key(models=[rig.paid], max_budget=BUDGET) denied: Final = fresh_chat(rig.gateway, rig.entry, outsider, "restricted-denied-" + uuid.uuid4().hex) assert _refusal(denied) == (403, "key_model_access_denied"), denied.text insider: Final = scenario.key(models=[rig.entry, rig.paid], max_budget=BUDGET) @@ -753,16 +757,21 @@ def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> No def test_repointing_the_chain_middle_follows_the_served_deployment(rig: ChainRig) -> None: with rig.gateway.scenario() as scenario: key: Final = _exhausted_key(rig, scenario) - scenario.cleanups.callback(install_aliases, rig.gateway, {rig.entry: rig.mid_free, rig.mid_free: rig.paid}) + scenario.cleanups.callback( + install_aliases, rig.gateway, {rig.entry: f"{rig.entry}-middle", rig.mid_free: rig.paid} + ) + install_aliases(rig.gateway, {rig.entry: rig.mid_free}) install_aliases(rig.gateway, {rig.mid_free: rig.tail_free}) _settle_both(rig, rig.mid_free, key, 200) _settle_both(rig, rig.entry, key, 200) install_aliases(rig.gateway, {rig.mid_free: rig.paid}) _settle_both(rig, rig.mid_free, key, BUDGET_EXCEEDED) - _settle_both(rig, rig.entry, key, 200) + _settle_both(rig, rig.entry, key, BUDGET_EXCEEDED) install_aliases(rig.gateway, {rig.entry: rig.paid}) _settle_both(rig, rig.entry, key, BUDGET_EXCEEDED) install_aliases(rig.gateway, {rig.entry: rig.mid_free}) + _settle_both(rig, rig.entry, key, BUDGET_EXCEEDED) + install_aliases(rig.gateway, {rig.entry: f"{rig.entry}-middle"}) _settle_both(rig, rig.entry, key, 200) @@ -942,8 +951,9 @@ def test_killing_one_worker_mid_burst_keeps_the_chain_entry_served(rig: ChainRig input_cost_per_token=0, output_cost_per_token=0, ) - install_aliases(rig.gateway, {entry: chaos_free, chaos_free: rig.paid}) - scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, chaos_free})) + middle: Final = f"{entry}-middle" + install_aliases(rig.gateway, {entry: middle, middle: chaos_free}) + scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle})) key: Final = _exhausted_key(rig, scenario) with owned_proxy_process(rig.gateway, tmp_path, {}, workers=2) as owned: candidate: Final = owned.gateway diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..9ce4739a2bc 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -2020,10 +2020,6 @@ async def test_get_user_object_cache_miss_emits_exactly_one_postgres_get_user_ob def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_routes_to( monkeypatch: pytest.MonkeyPatch, entry_first: bool ) -> None: - """chain-entry resolves one hop to local-free and is served by local-free's own free - deployment, so an over-budget key is waived for it; local-free by name resolves to paid-gpt - and stays enforced. local-free's alias is a hop the router never takes for chain-entry, and - the per-name verdict cache must not let either name's verdict leak into the other's.""" monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) router: Final = litellm.Router( model_list=[ @@ -2048,7 +2044,7 @@ def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_ro ], model_group_alias={"chain-entry": "local-free", "local-free": "paid-gpt"}, ) - expected: Final = {"chain-entry": True, "local-free": False, "paid-gpt": False} + expected: Final = {"chain-entry": False, "local-free": False, "paid-gpt": False} order: Final = ("chain-entry", "local-free", "paid-gpt") if entry_first else ("local-free", "paid-gpt", "chain-entry") verdicts: Final = {name: is_model_cost_zero(model=name, llm_router=router) for name in order} diff --git a/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py b/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py index c2851f608f0..830efab1d38 100644 --- a/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py +++ b/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py @@ -581,12 +581,7 @@ class TestUnmappedModelBudgetEnforcement: assert is_model_cost_zero(model="free-model", llm_router=router) is False - def test_alias_chain_through_a_priced_group_enforces_budget(self): - """An alias to a group that is itself an alias key resolves one hop, like the router does. - - The router serves ``chain-smart`` with the real ``chain-legacy`` deployment, which is priced, - so following the second hop to the free group would waive the budget for a paid call. - """ + def test_alias_chain_through_a_priced_group_to_a_free_target_bypasses_budget(self): router = Router( model_list=[ { @@ -613,18 +608,11 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={"chain-smart": "chain-legacy", "chain-legacy": "free-model"}, ) - assert is_model_cost_zero(model="chain-smart", llm_router=router) is False + assert _served_model(router, "chain-smart") == "ollama/llama2" + assert is_model_cost_zero(model="chain-smart", llm_router=router) is True @pytest.mark.parametrize("hidden", [False, True], ids=["plain_alias", "hidden_alias"]) - def test_alias_to_an_unpriced_group_that_is_also_an_alias_enforces_budget(self, hidden: bool): - """An alias is judged by the group it routes to, never by where that group's own alias points. - - The router serves ``chain-entry`` from the real ``chain-middle`` deployment, whose zero price - comes from the cost map alone. ``chain-middle`` is also an alias key to an explicitly free - group, a second hop the router never takes for ``chain-entry``, so that group must not lend - it the bypass. - """ - + def test_alias_chain_through_an_unpriced_group_uses_the_explicitly_free_target(self, hidden: bool): def alias(target: str) -> str | dict[str, str | bool]: return {"model": target, "hidden": True} if hidden else target @@ -633,19 +621,11 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={"chain-entry": alias("chain-middle"), "chain-middle": alias("free-model")}, ) - assert _served_model(router, "chain-entry") == UNPRICED_ZERO_COST_MODEL - assert is_model_cost_zero(model="chain-entry", llm_router=router) is False - assert is_model_cost_zero(model="chain-middle", llm_router=router) is True, ( - "asked by its own name, chain-middle routes to the explicitly free group" - ) + assert _served_model(router, "chain-entry") == "gpt-3.5-turbo" + assert is_model_cost_zero(model="chain-entry", llm_router=router) is True + assert is_model_cost_zero(model="chain-middle", llm_router=router) is True - def test_alias_to_a_free_group_that_is_also_an_alias_to_a_ptu_group_bypasses_budget(self): - """A PTU group one alias hop past the group that serves the request does not enforce the budget. - - The router serves ``chain-entry`` from the real, explicitly free ``chain-middle`` deployment. - ``chain-middle`` is also an alias key to a PTU-priced group, which only a request for - ``chain-middle`` itself routes to. - """ + def test_alias_chain_through_a_free_group_to_a_ptu_target_enforces_budget(self): router = Router( model_list=[ _explicitly_free("chain-middle"), @@ -654,19 +634,11 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={"chain-entry": "chain-middle", "chain-middle": "ptu-model"}, ) - assert _served_model(router, "chain-entry") == "gpt-3.5-turbo" - assert is_model_cost_zero(model="chain-entry", llm_router=router) is True - assert is_model_cost_zero(model="chain-middle", llm_router=router) is False, ( - "asked by its own name, chain-middle routes to the PTU group" - ) + assert _served_model(router, "chain-entry") == "azure/ptu-deployment" + assert is_model_cost_zero(model="chain-entry", llm_router=router) is False + assert is_model_cost_zero(model="chain-middle", llm_router=router) is False - def test_alias_chain_served_by_a_paid_wildcard_route_enforces_budget(self): - """An alias whose target is only an alias key is never judged by that second alias's free group. - - ``chain-entry`` resolves one hop to ``gpt-4o-mini``, which is no deployment's name, so the - router serves it from the paid wildcard route. The free group ``gpt-4o-mini`` is aliased - to is only reached by a request for ``gpt-4o-mini`` itself. - """ + def test_alias_chain_to_a_free_target_takes_precedence_over_a_paid_wildcard(self): router = Router( model_list=[ _explicitly_free("free-model", model="ollama/llama2"), @@ -675,17 +647,11 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={"chain-entry": "gpt-4o-mini", "gpt-4o-mini": "free-model"}, ) - assert _served_model(router, "chain-entry") == "openai/gpt-4o-mini" - assert is_model_cost_zero(model="chain-entry", llm_router=router) is False + assert _served_model(router, "chain-entry") == "ollama/llama2" + assert is_model_cost_zero(model="chain-entry", llm_router=router) is True @pytest.mark.parametrize("alias_name", ["openai/smart", "smart"], ids=["alias_on_pattern", "alias_off_pattern"]) - def test_alias_chain_served_by_an_explicitly_priced_wildcard_route_enforces_budget(self, alias_name: str): - """A chain the router serves from a priced wildcard route is budgeted at that route's price. - - The group's price reads $0 through the second alias to the free group, and the wildcard - route's cost-map entry carries explicit prices, so only their sign tells that the served - deployment is not free. - """ + def test_alias_chain_to_a_free_target_takes_precedence_over_an_explicitly_priced_wildcard(self, alias_name: str): router = Router( model_list=[ _explicitly_free("free-model", model="ollama/llama2"), @@ -703,11 +669,10 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={alias_name: "openai/gpt-4o-mini", "openai/gpt-4o-mini": "free-model"}, ) - assert _served_model(router, alias_name) == "openai/gpt-4o-mini" - assert is_model_cost_zero(model=alias_name, llm_router=router) is False + assert _served_model(router, alias_name) == "ollama/llama2" + assert is_model_cost_zero(model=alias_name, llm_router=router) is True - def test_alias_shadowing_a_free_group_is_judged_by_its_unpriced_target_through_an_alias_chain(self): - """A shadowing alias stays enforced when its unpriced target is itself an alias key to a free group.""" + def test_alias_shadowing_a_free_group_is_judged_by_the_final_target_through_an_unpriced_group(self): router = Router( model_list=[ _explicitly_free("shadowed-free"), @@ -717,8 +682,8 @@ class TestUnmappedModelBudgetEnforcement: model_group_alias={"shadowed-free": "chain-middle", "chain-middle": "free-model"}, ) - assert _served_model(router, "shadowed-free") == UNPRICED_ZERO_COST_MODEL - assert is_model_cost_zero(model="shadowed-free", llm_router=router) is False + assert _served_model(router, "shadowed-free") == "gpt-3.5-turbo" + assert is_model_cost_zero(model="shadowed-free", llm_router=router) is True @pytest.mark.parametrize("alias_name", ["ollama/fast", "fast"], ids=["alias_on_pattern", "alias_off_pattern"]) def test_alias_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self, alias_name: str): @@ -732,23 +697,15 @@ class TestUnmappedModelBudgetEnforcement: assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is True assert is_model_cost_zero(model=alias_name, llm_router=router) is True - def test_alias_chain_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self): - """An alias to an alias key no deployment is named after reads the wildcard route serving it. - - ``ollama/fast`` resolves one hop to ``ollama/llama3``, which is only an alias key, so the - router serves it from the explicitly free wildcard route. The group ``ollama/llama3`` is - aliased to is only reached by a request for ``ollama/llama3`` itself. - """ + def test_alias_chain_to_an_unpriced_target_does_not_inherit_a_free_wildcards_budget_waiver(self): router = Router( model_list=[_explicitly_free_ollama_wildcard(), _free_by_cost_map_only("unpriced-model")], model_group_alias={"ollama/fast": "ollama/llama3", "ollama/llama3": "unpriced-model"}, ) - assert _served_model(router, "ollama/fast") == "ollama/llama3" - assert is_model_cost_zero(model="ollama/fast", llm_router=router) is True - assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False, ( - "asked by its own name, ollama/llama3 routes to the unpriced group" - ) + assert _served_model(router, "ollama/fast") == UNPRICED_ZERO_COST_MODEL + assert is_model_cost_zero(model="ollama/fast", llm_router=router) is False + assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False def test_handles_router_without_zero_cost_cache_attribute(self): """Tolerate router-like objects (e.g. ``MagicMock`` stand-ins) that diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 82dd3479321..7d1e992ea81 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9567,6 +9567,8 @@ class TestPerRequestModelGroupAlias: ({"group-a": {"hidden": True}}, None), ({}, None), (None, None), + ({"group-a": "middle-alias", "middle-alias": {"model": "group-b", "hidden": True}}, "group-b"), + ({"group-a": "middle-alias", "middle-alias": "group-a"}, None), ], ) async def test_resolves_alias_for_the_requested_model_group(self, alias_map, expected): @@ -9580,13 +9582,20 @@ class TestPerRequestModelGroupAlias: assert resolved == expected @pytest.mark.asyncio - async def test_alias_target_outside_the_key_allowlist_is_rejected(self): + @pytest.mark.parametrize( + "aliases", + ( + {"group-a": "group-b"}, + {"group-a": "middle-alias", "middle-alias": "group-b"}, + ), + ) + async def test_alias_target_outside_the_key_allowlist_is_rejected(self, aliases: Mapping[str, str]) -> None: """Access was authorized against the requested group, so a rewrite that the key could not have requested directly must not be served.""" with pytest.raises(ProxyException) as exc_info: await _resolve_per_request_model_group_alias( requested_model="group-a", - router_settings={"model_group_alias": {"group-a": "group-b"}}, + router_settings={"model_group_alias": aliases}, user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a"]), llm_router=self._router(), ) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 0c9d7ab9f85..921d72ef2ea 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3018,7 +3018,6 @@ THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=Non OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None) FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1") SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None) -ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1") PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None) EVERY_DEPLOYMENT: Final = ( "refused-1", @@ -3142,8 +3141,8 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD) ("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None), ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), ("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), - ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)), - ("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", ALL_COOLED), ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), ("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), ), diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 3532870df1c..e492076684d 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -565,11 +565,27 @@ class TestResolveModelGroupAlias: ({"group-a": 5}, None), ({"group-a": None}, None), ({"group-a": ""}, None), + ({"group-a": "group-b", "group-b": "group-c"}, "group-c"), + ( + {"group-a": {"model": "group-b"}, "group-b": {"model": "group-c", "hidden": True}}, + "group-c", + ), + ({"group-a": "group-b", "group-b": {"hidden": True}}, "group-b"), + ({"group-a": "group-a"}, None), + ({"group-a": "group-b", "group-b": "group-a"}, None), + ({"group-a": "group-b", "group-b": "group-c", "group-c": "group-b"}, None), ], ) - def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected): + def test_resolves_both_entry_shapes_and_tolerates_malformed_entries( + self, alias_map: object, expected: str | None + ) -> None: assert resolve_model_group_alias(alias_map, "group-a") == expected + def test_resolves_alias_chains_longer_than_the_python_recursion_limit(self) -> None: + aliases: Final = {f"group-{index}": f"group-{index + 1}" for index in range(1100)} + + assert resolve_model_group_alias(aliases, "group-0") == "group-1100" + def test_router_alias_resolution_uses_the_shared_helper(self): router = Router( model_list=[ @@ -593,6 +609,7 @@ class TestResolveModelGroupAlias: pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"), pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"), pytest.param("a", {}, {}, {}, "a", id="no-alias"), + pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d", "d": "e"}, "e", id="router-alias-chain"), ], ) def test_resolves_served_model_aliases_in_order( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..1e9132bf7f4 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2742,9 +2742,6 @@ def _assert_free(info: ModelGroupInfo | None, provider: str) -> None: def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to(): - """chain-entry resolves one hop to local-free and is served by local-free's own - deployment, so its price is that deployment's; local-free's own alias to gpt-priced - is a hop the router takes only for a request to local-free by name.""" router = Router( model_list=[ _free_ollama_deployment("local-free"), @@ -2753,14 +2750,11 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to( model_group_alias={"chain-entry": "local-free", "local-free": "gpt-priced"}, ) - _assert_free(router.get_model_group_info(model_group="chain-entry"), "ollama") + _assert_priced(router.get_model_group_info(model_group="chain-entry"), "openai") _assert_priced(router.get_model_group_info(model_group="local-free"), "openai") def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serving_it(): - """When the routed group has no deployment of its own, the wildcard route matching it - serves the request, so the price is the wildcard's and never the routed group's own alias - target's.""" router = Router( model_list=[ _paid_openai_deployment("openai/*", "openai/*"), @@ -2769,7 +2763,7 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serv model_group_alias={"wildcard-entry": "openai/gpt-4o", "openai/gpt-4o": "local-free"}, ) - _assert_priced(router.get_model_group_info(model_group="wildcard-entry"), "openai") + _assert_free(router.get_model_group_info(model_group="wildcard-entry"), "ollama") _assert_free(router.get_model_group_info(model_group="openai/gpt-4o"), "ollama") @@ -2778,9 +2772,6 @@ def _served_models(deployments: list[DeploymentTypedDict] | None) -> list[str]: def test_get_model_list_of_routed_group_reads_the_groups_own_deployments_only(): - """The router resolves an alias once, so a group reached as an alias target is served by - its own deployments. get_model_list composes the group's own alias target too, the hop a - request to that group by name takes.""" router = Router( model_list=[ _free_ollama_deployment("local-free"), @@ -27560,6 +27551,36 @@ def test_router_get_model_list_from_model_alias() -> None: assert len(model_alias_list) == 0 +@pytest.mark.parametrize("hidden", (False, True)) +@pytest.mark.parametrize("use_async", (False, True)) +async def test_chained_model_group_alias_selects_the_final_deployment(hidden: bool, use_async: bool) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "concrete-model", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-fake"}, + "model_info": {"id": "concrete-deployment"}, + } + ], + model_group_alias={ + "public-alias": "middle-alias", + "middle-alias": {"model": "concrete-model", "hidden": hidden}, + }, + ) + + deployment: Final = ( + await router.async_get_available_deployment(model="public-alias", messages=[], request_kwargs={}) + if use_async + else router.get_available_deployment(model="public-alias", messages=[]) + ) + + assert deployment["model_info"]["id"] == "concrete-deployment" + assert router.routable_model_group("public-alias") == "concrete-model" + assert _served_models(router.get_model_list(model_name="public-alias")) == ["openai/gpt-6.1-sol"] + assert "public-alias" in router.get_model_names() + assert ("middle-alias" in router.get_model_names()) is not hidden + + def test_router_dynamic_credentials() -> None: """ Assert model id for dynamic api key 1 != model id for dynamic api key 2