This commit is contained in:
Elvin Chu 2026-10-27 02:24:21 +00:00 • committed by GitHub
commit 7f236eb562
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 157 additions and 167 deletions

View file

@ -664,23 +664,10 @@ def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[Deploy
"""
The deployments a request to ``model`` routes to, for the gates that waive budget checks.
The router resolves a ``model_group_alias`` name to its target exactly once and serves that
group, never a real deployment that shares the alias's name. ``Router.get_model_list()`` on the
routed group reads those deployments unless the group is itself an alias key, where it would
follow the alias a second hop the router never takes; then the deployments named after the
group count, or the wildcard route serving the group when none carries its name, the same
fallback ``get_model_list()`` takes.
Resolve the complete alias chain before looking up the served deployments.
"""
routed_group: Final = llm_router.routable_model_group(model)
if llm_router.routable_model_group(routed_group) == routed_group:
return llm_router.get_model_list(model_name=routed_group) or ()
named: Final = llm_router.get_model_list_from_model_alias(model_name=model)
if named:
return named
return tuple(
DeploymentTypedDict(**deployment)
for deployment in llm_router.pattern_router.get_deployments_by_pattern(model=routed_group)
)
return llm_router.get_model_list(model_name=routed_group) or ()
def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:

View file

@ -12740,14 +12740,10 @@ class Router:
## Check if model group alias
if model_group in self.model_group_alias:
item: Final = self.model_group_alias[model_group]
if isinstance(item, str):
_router_model_group = item
elif isinstance(item, dict):
if item["hidden"] is True and not include_hidden:
return None
else:
_router_model_group = item["model"]
else:
if isinstance(item, dict) and item.get("hidden") is True and not include_hidden:
return None
_router_model_group: Final = self.get_model_from_alias(model_group)
if _router_model_group is None:
return None
return self._set_model_group_info(
@ -13560,14 +13556,10 @@ class Router:
alias_items = list(self.model_group_alias.items())
for model_alias, model_value in alias_items:
if isinstance(model_value, str):
_router_model_name: str = model_value
elif isinstance(model_value, dict):
_model_value = RouterModelGroupAliasItem(**model_value)
if _model_value["hidden"] is True and model_name is None:
continue
_router_model_name = _model_value["model"]
else:
if isinstance(model_value, dict) and model_value.get("hidden") is True and model_name is None:
continue
_router_model_name = self.get_model_from_alias(model_alias)
if _router_model_name is None:
continue
if (alias_group := self.get_routing_group(_router_model_name)) is not None:
@ -13831,9 +13823,7 @@ class Router:
the wildcard route matching it when neither exists.
Unlike get_model_list, model_group's own model_group_alias entry is not
followed. The router resolves an alias exactly once, so a group reached as
an alias target is served by its own deployments, never by a second hop:
in the chain X -> T -> U a request to X is served from T's deployments.
followed. The caller has already resolved the complete alias chain.
"""
named: Final = [
*self._get_all_deployments(model_name=model_group),

View file

@ -103,15 +103,19 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No
entries: the map can come from a key or team row rather than from validated
config, so a bad value must not raise mid-request.
Returns the target model group, or None when the map does not rewrite ``model``.
Returns the final target model group, or None when no alias applies or the chain cycles.
"""
if not isinstance(model_group_alias, Mapping):
if not isinstance(model_group_alias, Mapping) or model not in model_group_alias:
return None
entry: Final = model_group_alias.get(model)
target: Final = entry.get("model") if isinstance(entry, Mapping) else entry
if not isinstance(target, str) or not target:
return None
return target
aliases: Final = _OBJECT_MAPPING_ADAPTER.validate_python(model_group_alias)
current: str = model # rebind-ok: iterative traversal supports chains beyond Python's recursion limit
for _ in range(len(aliases) + 1):
entry = aliases.get(current)
target = _OBJECT_MAPPING_ADAPTER.validate_python(entry).get("model") if isinstance(entry, Mapping) else entry
if not isinstance(target, str) or not target:
return current if current != model else None
current = target
return None
def resolve_served_model(

View file

@ -293,7 +293,7 @@ def _await_rig(rig: ChainRig) -> None:
tail_id: Final = _deployment_id(rig.tail_free)
for candidate in rig.candidates:
_await_served_by(candidate, rig.mid_free, paid_id)
_await_served_by(candidate, rig.failing_free, paid_id)
_await_served_by(candidate, rig.failing_free, _deployment_id(rig.failing_free))
_await_served_by(candidate, rig.mid_paid, tail_id)
_await_served_by(candidate, rig.unpriced, tail_id)
@ -331,11 +331,14 @@ def _chain_rig() -> Generator[ChainRig]:
ghost_entry=f"chain-ghost-entry-{suffix}",
failing_entry=f"chain-failing-entry-{suffix}",
)
free_middle: Final = f"{rig.entry}-middle"
failing_middle: Final = f"{rig.failing_entry}-middle"
aliases: Final[Mapping[str, JsonValue]] = {
rig.entry: rig.mid_free,
rig.hidden_entry: hidden(rig.mid_free),
rig.shown_entry: {"model": rig.mid_free, "hidden": False},
rig.null_entry: {"model": rig.mid_free, "hidden": None},
rig.entry: free_middle,
rig.hidden_entry: hidden(free_middle),
rig.shown_entry: {"model": free_middle, "hidden": False},
rig.null_entry: {"model": free_middle, "hidden": None},
free_middle: rig.tail_free,
rig.mid_free: rig.paid,
rig.reverse_entry: rig.mid_paid,
rig.mid_paid: rig.tail_free,
@ -343,8 +346,8 @@ def _chain_rig() -> Generator[ChainRig]:
rig.unpriced: rig.tail_free,
rig.ghost_entry: rig.ghost_mid,
rig.ghost_mid: rig.paid,
rig.failing_entry: rig.failing_free,
rig.failing_free: rig.paid,
rig.failing_entry: failing_middle,
failing_middle: rig.failing_free,
}
install_aliases(gateway, aliases)
scenario.cleanups.callback(remove_aliases, gateway, frozenset(aliases))
@ -524,46 +527,47 @@ def test_chain_middle_by_name_is_judged_by_its_own_alias(rig: ChainRig) -> None:
assert row["model_group"] == rig.mid_free, row
def test_reverse_chain_is_priced_by_the_middle_it_is_served_from(rig: ChainRig) -> None:
def test_chain_through_a_paid_middle_is_priced_by_its_free_target(rig: ChainRig) -> None:
free_marker: Final = "reverse-free-" + uuid.uuid4().hex
billed_marker: Final = "reverse-billed-" + uuid.uuid4().hex
entry_marker: Final = "reverse-entry-" + uuid.uuid4().hex
with rig.gateway.scenario() as scenario:
exhausted: Final = _exhausted_key(rig, scenario)
for candidate in rig.candidates:
_assert_refused_on_every_route(rig, candidate, exhausted, rig.reverse_entry)
_assert_raw_chat_served(rig, candidate, rig.reverse_entry, exhausted, "reverse-chain")
served: Final = fresh_chat(rig.gateway, rig.mid_paid, exhausted, free_marker)
assert served.status_code == 200, served.text
assert served.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict(served.headers)
assert_free_row(landed_once(exhausted, free_marker), rig.mid_paid)
budgeted: Final = scenario.key(max_budget=HEADROOM)
billed: Final = fresh_chat(rig.gateway, rig.reverse_entry, budgeted, billed_marker)
assert billed.status_code == 200, billed.text
assert billed.headers["x-litellm-model-id"] == _deployment_id(rig.mid_paid), dict(billed.headers)
row: Final = landed_once(budgeted, billed_marker)
assert math.isclose(float(str(row["spend"])), PAID_REPLY_COST), row
assert row["model_group"] == rig.reverse_entry, row
entry_response: Final = fresh_chat(rig.gateway, rig.reverse_entry, exhausted, entry_marker)
assert entry_response.status_code == 200, entry_response.text
assert entry_response.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict(
entry_response.headers
)
assert_free_row(landed_once(exhausted, entry_marker), rig.reverse_entry)
def test_chain_through_a_cost_map_priced_middle_stays_budgeted(rig: ChainRig) -> None:
def test_chain_through_a_cost_map_priced_middle_uses_the_free_targets_budget_waiver(rig: ChainRig) -> None:
with rig.gateway.scenario() as scenario:
key: Final = _exhausted_key(rig, scenario)
for candidate in rig.candidates:
_assert_refused_on_every_route(rig, candidate, key, rig.unpriced_entry)
_assert_raw_chat_served(rig, candidate, rig.unpriced_entry, key, "unpriced-middle")
def _assert_ghost_chain(rig: ChainRig, candidate: Gateway, key: str) -> None:
assert _group_rows(candidate, rig.ghost_entry) == ()
assert rig.ghost_entry not in _listed_input_prices(candidate)
_assert_price(
_price_of(candidate, rig.ghost_entry), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",)
)
assert _listed_input_prices(candidate)[rig.ghost_entry] == PAID_INPUT_COST_PER_TOKEN
refused: Final = fresh_chat(candidate, rig.ghost_entry, key, "ghost-exhausted-" + uuid.uuid4().hex)
assert refused.status_code == BUDGET_EXCEEDED, refused.text
unroutable: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex)
assert unroutable.status_code == 400, unroutable.text
assert "no healthy deployments" in unroutable.text, unroutable.text
served: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex)
assert served.status_code == 200, served.text
assert served.headers["x-litellm-model-id"] == _deployment_id(rig.paid), dict(served.headers)
for path in ("/health/liveliness", "/model/info", "/v1/models"):
assert candidate.request("GET", path).status_code == 200, path
def test_ghost_chain_reports_no_group_and_stays_unroutable(rig: ChainRig) -> None:
def test_chain_through_an_alias_without_deployments_routes_to_its_paid_target(rig: ChainRig) -> None:
with rig.gateway.scenario() as scenario:
key: Final = _exhausted_key(rig, scenario)
for candidate in rig.candidates:
@ -590,7 +594,7 @@ def test_provider_failure_behind_a_chain_entry_reaches_the_caller(rig: ChainRig)
_assert_no_served_row(key, failed_marker)
def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainRig) -> None:
def test_chain_to_a_paid_target_does_not_inherit_a_free_wildcards_budget_waiver(rig: ChainRig) -> None:
prefix: Final = "wc" + uuid.uuid4().hex[:8]
middle: Final = f"{prefix}/gpt-4o-mini"
other: Final = f"{prefix}/gpt-4o"
@ -601,7 +605,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
paid_id: Final = _deployment_id(rig.paid)
for candidate in rig.candidates:
_await_served_by(candidate, entry, route)
_await_served_by(candidate, entry, paid_id)
_await_served_by(candidate, middle, paid_id)
key: Final = _exhausted_key(rig, scenario)
for candidate in rig.candidates:
@ -610,7 +614,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR
_assert_price(
_price_of(candidate, middle), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",)
)
_assert_served_by_route(candidate, entry, key, route, "wildcard-served")
_assert_refused_on_every_route(rig, candidate, key, entry)
_assert_refused(rig, candidate, key, middle, "wildcard-middle")
@ -622,21 +626,21 @@ def _assert_served_by_route(candidate: Gateway, model: str, key: str, route: str
assert_free_row(landed_once(key, marker), model)
def test_chain_through_a_priced_wildcard_route_stays_budgeted(rig: ChainRig) -> None:
def test_chain_to_a_free_target_takes_precedence_over_a_priced_wildcard_route(rig: ChainRig) -> None:
prefix: Final = "pwc" + uuid.uuid4().hex[:8]
middle: Final = f"{prefix}/gpt-4o-mini"
entry: Final = "priced-wildcard-entry-" + uuid.uuid4().hex
with rig.gateway.scenario() as scenario:
route: Final = _wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN)
_wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN)
install_aliases(rig.gateway, {entry: middle, middle: rig.tail_free})
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
tail_id: Final = _deployment_id(rig.tail_free)
for candidate in rig.candidates:
_await_served_by(candidate, entry, route)
_await_served_by(candidate, entry, tail_id)
_await_served_by(candidate, middle, tail_id)
key: Final = _exhausted_key(rig, scenario)
for candidate in rig.candidates:
_assert_refused_on_every_route(rig, candidate, key, entry)
_assert_raw_chat_served(rig, candidate, entry, key, "priced-wildcard-free-target")
def _assert_hostile_group_queries_answered(rig: ChainRig, candidate: Gateway) -> None:
@ -738,7 +742,7 @@ def test_duplicate_model_field_is_judged_by_its_last_value(rig: ChainRig) -> Non
def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> None:
marker: Final = "restricted-served-" + uuid.uuid4().hex
with rig.gateway.scenario() as scenario:
outsider: Final = scenario.key(models=[rig.tail_free, rig.paid], max_budget=BUDGET)
outsider: Final = scenario.key(models=[rig.paid], max_budget=BUDGET)
denied: Final = fresh_chat(rig.gateway, rig.entry, outsider, "restricted-denied-" + uuid.uuid4().hex)
assert _refusal(denied) == (403, "key_model_access_denied"), denied.text
insider: Final = scenario.key(models=[rig.entry, rig.paid], max_budget=BUDGET)
@ -753,16 +757,21 @@ def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> No
def test_repointing_the_chain_middle_follows_the_served_deployment(rig: ChainRig) -> None:
with rig.gateway.scenario() as scenario:
key: Final = _exhausted_key(rig, scenario)
scenario.cleanups.callback(install_aliases, rig.gateway, {rig.entry: rig.mid_free, rig.mid_free: rig.paid})
scenario.cleanups.callback(
install_aliases, rig.gateway, {rig.entry: f"{rig.entry}-middle", rig.mid_free: rig.paid}
)
install_aliases(rig.gateway, {rig.entry: rig.mid_free})
install_aliases(rig.gateway, {rig.mid_free: rig.tail_free})
_settle_both(rig, rig.mid_free, key, 200)
_settle_both(rig, rig.entry, key, 200)
install_aliases(rig.gateway, {rig.mid_free: rig.paid})
_settle_both(rig, rig.mid_free, key, BUDGET_EXCEEDED)
_settle_both(rig, rig.entry, key, 200)
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
install_aliases(rig.gateway, {rig.entry: rig.paid})
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
install_aliases(rig.gateway, {rig.entry: rig.mid_free})
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
install_aliases(rig.gateway, {rig.entry: f"{rig.entry}-middle"})
_settle_both(rig, rig.entry, key, 200)
@ -942,8 +951,9 @@ def test_killing_one_worker_mid_burst_keeps_the_chain_entry_served(rig: ChainRig
input_cost_per_token=0,
output_cost_per_token=0,
)
install_aliases(rig.gateway, {entry: chaos_free, chaos_free: rig.paid})
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, chaos_free}))
middle: Final = f"{entry}-middle"
install_aliases(rig.gateway, {entry: middle, middle: chaos_free})
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
key: Final = _exhausted_key(rig, scenario)
with owned_proxy_process(rig.gateway, tmp_path, {}, workers=2) as owned:
candidate: Final = owned.gateway

View file

@ -2020,10 +2020,6 @@ async def test_get_user_object_cache_miss_emits_exactly_one_postgres_get_user_ob
def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_routes_to(
monkeypatch: pytest.MonkeyPatch, entry_first: bool
) -> None:
"""chain-entry resolves one hop to local-free and is served by local-free's own free
deployment, so an over-budget key is waived for it; local-free by name resolves to paid-gpt
and stays enforced. local-free's alias is a hop the router never takes for chain-entry, and
the per-name verdict cache must not let either name's verdict leak into the other's."""
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
router: Final = litellm.Router(
model_list=[
@ -2048,7 +2044,7 @@ def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_ro
],
model_group_alias={"chain-entry": "local-free", "local-free": "paid-gpt"},
)
expected: Final = {"chain-entry": True, "local-free": False, "paid-gpt": False}
expected: Final = {"chain-entry": False, "local-free": False, "paid-gpt": False}
order: Final = ("chain-entry", "local-free", "paid-gpt") if entry_first else ("local-free", "paid-gpt", "chain-entry")
verdicts: Final = {name: is_model_cost_zero(model=name, llm_router=router) for name in order}

View file

@ -581,12 +581,7 @@ class TestUnmappedModelBudgetEnforcement:
assert is_model_cost_zero(model="free-model", llm_router=router) is False
def test_alias_chain_through_a_priced_group_enforces_budget(self):
"""An alias to a group that is itself an alias key resolves one hop, like the router does.
The router serves ``chain-smart`` with the real ``chain-legacy`` deployment, which is priced,
so following the second hop to the free group would waive the budget for a paid call.
"""
def test_alias_chain_through_a_priced_group_to_a_free_target_bypasses_budget(self):
router = Router(
model_list=[
{
@ -613,18 +608,11 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={"chain-smart": "chain-legacy", "chain-legacy": "free-model"},
)
assert is_model_cost_zero(model="chain-smart", llm_router=router) is False
assert _served_model(router, "chain-smart") == "ollama/llama2"
assert is_model_cost_zero(model="chain-smart", llm_router=router) is True
@pytest.mark.parametrize("hidden", [False, True], ids=["plain_alias", "hidden_alias"])
def test_alias_to_an_unpriced_group_that_is_also_an_alias_enforces_budget(self, hidden: bool):
"""An alias is judged by the group it routes to, never by where that group's own alias points.
The router serves ``chain-entry`` from the real ``chain-middle`` deployment, whose zero price
comes from the cost map alone. ``chain-middle`` is also an alias key to an explicitly free
group, a second hop the router never takes for ``chain-entry``, so that group must not lend
it the bypass.
"""
def test_alias_chain_through_an_unpriced_group_uses_the_explicitly_free_target(self, hidden: bool):
def alias(target: str) -> str | dict[str, str | bool]:
return {"model": target, "hidden": True} if hidden else target
@ -633,19 +621,11 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={"chain-entry": alias("chain-middle"), "chain-middle": alias("free-model")},
)
assert _served_model(router, "chain-entry") == UNPRICED_ZERO_COST_MODEL
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
assert is_model_cost_zero(model="chain-middle", llm_router=router) is True, (
"asked by its own name, chain-middle routes to the explicitly free group"
)
assert _served_model(router, "chain-entry") == "gpt-3.5-turbo"
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
assert is_model_cost_zero(model="chain-middle", llm_router=router) is True
def test_alias_to_a_free_group_that_is_also_an_alias_to_a_ptu_group_bypasses_budget(self):
"""A PTU group one alias hop past the group that serves the request does not enforce the budget.
The router serves ``chain-entry`` from the real, explicitly free ``chain-middle`` deployment.
``chain-middle`` is also an alias key to a PTU-priced group, which only a request for
``chain-middle`` itself routes to.
"""
def test_alias_chain_through_a_free_group_to_a_ptu_target_enforces_budget(self):
router = Router(
model_list=[
_explicitly_free("chain-middle"),
@ -654,19 +634,11 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={"chain-entry": "chain-middle", "chain-middle": "ptu-model"},
)
assert _served_model(router, "chain-entry") == "gpt-3.5-turbo"
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
assert is_model_cost_zero(model="chain-middle", llm_router=router) is False, (
"asked by its own name, chain-middle routes to the PTU group"
)
assert _served_model(router, "chain-entry") == "azure/ptu-deployment"
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
assert is_model_cost_zero(model="chain-middle", llm_router=router) is False
def test_alias_chain_served_by_a_paid_wildcard_route_enforces_budget(self):
"""An alias whose target is only an alias key is never judged by that second alias's free group.
``chain-entry`` resolves one hop to ``gpt-4o-mini``, which is no deployment's name, so the
router serves it from the paid wildcard route. The free group ``gpt-4o-mini`` is aliased
to is only reached by a request for ``gpt-4o-mini`` itself.
"""
def test_alias_chain_to_a_free_target_takes_precedence_over_a_paid_wildcard(self):
router = Router(
model_list=[
_explicitly_free("free-model", model="ollama/llama2"),
@ -675,17 +647,11 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={"chain-entry": "gpt-4o-mini", "gpt-4o-mini": "free-model"},
)
assert _served_model(router, "chain-entry") == "openai/gpt-4o-mini"
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
assert _served_model(router, "chain-entry") == "ollama/llama2"
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
@pytest.mark.parametrize("alias_name", ["openai/smart", "smart"], ids=["alias_on_pattern", "alias_off_pattern"])
def test_alias_chain_served_by_an_explicitly_priced_wildcard_route_enforces_budget(self, alias_name: str):
"""A chain the router serves from a priced wildcard route is budgeted at that route's price.
The group's price reads $0 through the second alias to the free group, and the wildcard
route's cost-map entry carries explicit prices, so only their sign tells that the served
deployment is not free.
"""
def test_alias_chain_to_a_free_target_takes_precedence_over_an_explicitly_priced_wildcard(self, alias_name: str):
router = Router(
model_list=[
_explicitly_free("free-model", model="ollama/llama2"),
@ -703,11 +669,10 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={alias_name: "openai/gpt-4o-mini", "openai/gpt-4o-mini": "free-model"},
)
assert _served_model(router, alias_name) == "openai/gpt-4o-mini"
assert is_model_cost_zero(model=alias_name, llm_router=router) is False
assert _served_model(router, alias_name) == "ollama/llama2"
assert is_model_cost_zero(model=alias_name, llm_router=router) is True
def test_alias_shadowing_a_free_group_is_judged_by_its_unpriced_target_through_an_alias_chain(self):
"""A shadowing alias stays enforced when its unpriced target is itself an alias key to a free group."""
def test_alias_shadowing_a_free_group_is_judged_by_the_final_target_through_an_unpriced_group(self):
router = Router(
model_list=[
_explicitly_free("shadowed-free"),
@ -717,8 +682,8 @@ class TestUnmappedModelBudgetEnforcement:
model_group_alias={"shadowed-free": "chain-middle", "chain-middle": "free-model"},
)
assert _served_model(router, "shadowed-free") == UNPRICED_ZERO_COST_MODEL
assert is_model_cost_zero(model="shadowed-free", llm_router=router) is False
assert _served_model(router, "shadowed-free") == "gpt-3.5-turbo"
assert is_model_cost_zero(model="shadowed-free", llm_router=router) is True
@pytest.mark.parametrize("alias_name", ["ollama/fast", "fast"], ids=["alias_on_pattern", "alias_off_pattern"])
def test_alias_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self, alias_name: str):
@ -732,23 +697,15 @@ class TestUnmappedModelBudgetEnforcement:
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is True
assert is_model_cost_zero(model=alias_name, llm_router=router) is True
def test_alias_chain_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self):
"""An alias to an alias key no deployment is named after reads the wildcard route serving it.
``ollama/fast`` resolves one hop to ``ollama/llama3``, which is only an alias key, so the
router serves it from the explicitly free wildcard route. The group ``ollama/llama3`` is
aliased to is only reached by a request for ``ollama/llama3`` itself.
"""
def test_alias_chain_to_an_unpriced_target_does_not_inherit_a_free_wildcards_budget_waiver(self):
router = Router(
model_list=[_explicitly_free_ollama_wildcard(), _free_by_cost_map_only("unpriced-model")],
model_group_alias={"ollama/fast": "ollama/llama3", "ollama/llama3": "unpriced-model"},
)
assert _served_model(router, "ollama/fast") == "ollama/llama3"
assert is_model_cost_zero(model="ollama/fast", llm_router=router) is True
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False, (
"asked by its own name, ollama/llama3 routes to the unpriced group"
)
assert _served_model(router, "ollama/fast") == UNPRICED_ZERO_COST_MODEL
assert is_model_cost_zero(model="ollama/fast", llm_router=router) is False
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False
def test_handles_router_without_zero_cost_cache_attribute(self):
"""Tolerate router-like objects (e.g. ``MagicMock`` stand-ins) that

View file

@ -9567,6 +9567,8 @@ class TestPerRequestModelGroupAlias:
({"group-a": {"hidden": True}}, None),
({}, None),
(None, None),
({"group-a": "middle-alias", "middle-alias": {"model": "group-b", "hidden": True}}, "group-b"),
({"group-a": "middle-alias", "middle-alias": "group-a"}, None),
],
)
async def test_resolves_alias_for_the_requested_model_group(self, alias_map, expected):
@ -9580,13 +9582,20 @@ class TestPerRequestModelGroupAlias:
assert resolved == expected
@pytest.mark.asyncio
async def test_alias_target_outside_the_key_allowlist_is_rejected(self):
@pytest.mark.parametrize(
"aliases",
(
{"group-a": "group-b"},
{"group-a": "middle-alias", "middle-alias": "group-b"},
),
)
async def test_alias_target_outside_the_key_allowlist_is_rejected(self, aliases: Mapping[str, str]) -> None:
"""Access was authorized against the requested group, so a rewrite that
the key could not have requested directly must not be served."""
with pytest.raises(ProxyException) as exc_info:
await _resolve_per_request_model_group_alias(
requested_model="group-a",
router_settings={"model_group_alias": {"group-a": "group-b"}},
router_settings={"model_group_alias": aliases},
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a"]),
llm_router=self._router(),
)

View file

@ -3018,7 +3018,6 @@ THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=Non
OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None)
FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1")
SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None)
ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1")
PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None)
EVERY_DEPLOYMENT: Final = (
"refused-1",
@ -3142,8 +3141,8 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD)
("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", ALL_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", ALL_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)),
),

View file

@ -565,11 +565,27 @@ class TestResolveModelGroupAlias:
({"group-a": 5}, None),
({"group-a": None}, None),
({"group-a": ""}, None),
({"group-a": "group-b", "group-b": "group-c"}, "group-c"),
(
{"group-a": {"model": "group-b"}, "group-b": {"model": "group-c", "hidden": True}},
"group-c",
),
({"group-a": "group-b", "group-b": {"hidden": True}}, "group-b"),
({"group-a": "group-a"}, None),
({"group-a": "group-b", "group-b": "group-a"}, None),
({"group-a": "group-b", "group-b": "group-c", "group-c": "group-b"}, None),
],
)
def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected):
def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(
self, alias_map: object, expected: str | None
) -> None:
assert resolve_model_group_alias(alias_map, "group-a") == expected
def test_resolves_alias_chains_longer_than_the_python_recursion_limit(self) -> None:
aliases: Final = {f"group-{index}": f"group-{index + 1}" for index in range(1100)}
assert resolve_model_group_alias(aliases, "group-0") == "group-1100"
def test_router_alias_resolution_uses_the_shared_helper(self):
router = Router(
model_list=[
@ -593,6 +609,7 @@ class TestResolveModelGroupAlias:
pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"),
pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"),
pytest.param("a", {}, {}, {}, "a", id="no-alias"),
pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d", "d": "e"}, "e", id="router-alias-chain"),
],
)
def test_resolves_served_model_aliases_in_order(

View file

@ -2742,9 +2742,6 @@ def _assert_free(info: ModelGroupInfo | None, provider: str) -> None:
def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to():
"""chain-entry resolves one hop to local-free and is served by local-free's own
deployment, so its price is that deployment's; local-free's own alias to gpt-priced
is a hop the router takes only for a request to local-free by name."""
router = Router(
model_list=[
_free_ollama_deployment("local-free"),
@ -2753,14 +2750,11 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to(
model_group_alias={"chain-entry": "local-free", "local-free": "gpt-priced"},
)
_assert_free(router.get_model_group_info(model_group="chain-entry"), "ollama")
_assert_priced(router.get_model_group_info(model_group="chain-entry"), "openai")
_assert_priced(router.get_model_group_info(model_group="local-free"), "openai")
def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serving_it():
"""When the routed group has no deployment of its own, the wildcard route matching it
serves the request, so the price is the wildcard's and never the routed group's own alias
target's."""
router = Router(
model_list=[
_paid_openai_deployment("openai/*", "openai/*"),
@ -2769,7 +2763,7 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serv
model_group_alias={"wildcard-entry": "openai/gpt-4o", "openai/gpt-4o": "local-free"},
)
_assert_priced(router.get_model_group_info(model_group="wildcard-entry"), "openai")
_assert_free(router.get_model_group_info(model_group="wildcard-entry"), "ollama")
_assert_free(router.get_model_group_info(model_group="openai/gpt-4o"), "ollama")
@ -2778,9 +2772,6 @@ def _served_models(deployments: list[DeploymentTypedDict] | None) -> list[str]:
def test_get_model_list_of_routed_group_reads_the_groups_own_deployments_only():
"""The router resolves an alias once, so a group reached as an alias target is served by
its own deployments. get_model_list composes the group's own alias target too, the hop a
request to that group by name takes."""
router = Router(
model_list=[
_free_ollama_deployment("local-free"),
@ -27560,6 +27551,36 @@ def test_router_get_model_list_from_model_alias() -> None:
assert len(model_alias_list) == 0
@pytest.mark.parametrize("hidden", (False, True))
@pytest.mark.parametrize("use_async", (False, True))
async def test_chained_model_group_alias_selects_the_final_deployment(hidden: bool, use_async: bool) -> None:
router: Final = Router(
model_list=[
{
"model_name": "concrete-model",
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-fake"},
"model_info": {"id": "concrete-deployment"},
}
],
model_group_alias={
"public-alias": "middle-alias",
"middle-alias": {"model": "concrete-model", "hidden": hidden},
},
)
deployment: Final = (
await router.async_get_available_deployment(model="public-alias", messages=[], request_kwargs={})
if use_async
else router.get_available_deployment(model="public-alias", messages=[])
)
assert deployment["model_info"]["id"] == "concrete-deployment"
assert router.routable_model_group("public-alias") == "concrete-model"
assert _served_models(router.get_model_list(model_name="public-alias")) == ["openai/gpt-6.1-sol"]
assert "public-alias" in router.get_model_names()
assert ("middle-alias" in router.get_model_names()) is not hidden
def test_router_dynamic_credentials() -> None:
"""
Assert model id for dynamic api key 1 != model id for dynamic api key 2