mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 999bc9fc9e into e16d79d635
This commit is contained in:
commit
7f236eb562
10 changed files with 157 additions and 167 deletions
|
|
@ -664,23 +664,10 @@ def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[Deploy
|
|||
"""
|
||||
The deployments a request to ``model`` routes to, for the gates that waive budget checks.
|
||||
|
||||
The router resolves a ``model_group_alias`` name to its target exactly once and serves that
|
||||
group, never a real deployment that shares the alias's name. ``Router.get_model_list()`` on the
|
||||
routed group reads those deployments unless the group is itself an alias key, where it would
|
||||
follow the alias a second hop the router never takes; then the deployments named after the
|
||||
group count, or the wildcard route serving the group when none carries its name, the same
|
||||
fallback ``get_model_list()`` takes.
|
||||
Resolve the complete alias chain before looking up the served deployments.
|
||||
"""
|
||||
routed_group: Final = llm_router.routable_model_group(model)
|
||||
if llm_router.routable_model_group(routed_group) == routed_group:
|
||||
return llm_router.get_model_list(model_name=routed_group) or ()
|
||||
named: Final = llm_router.get_model_list_from_model_alias(model_name=model)
|
||||
if named:
|
||||
return named
|
||||
return tuple(
|
||||
DeploymentTypedDict(**deployment)
|
||||
for deployment in llm_router.pattern_router.get_deployments_by_pattern(model=routed_group)
|
||||
)
|
||||
return llm_router.get_model_list(model_name=routed_group) or ()
|
||||
|
||||
|
||||
def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
|
||||
|
|
|
|||
|
|
@ -12740,14 +12740,10 @@ class Router:
|
|||
## Check if model group alias
|
||||
if model_group in self.model_group_alias:
|
||||
item: Final = self.model_group_alias[model_group]
|
||||
if isinstance(item, str):
|
||||
_router_model_group = item
|
||||
elif isinstance(item, dict):
|
||||
if item["hidden"] is True and not include_hidden:
|
||||
return None
|
||||
else:
|
||||
_router_model_group = item["model"]
|
||||
else:
|
||||
if isinstance(item, dict) and item.get("hidden") is True and not include_hidden:
|
||||
return None
|
||||
_router_model_group: Final = self.get_model_from_alias(model_group)
|
||||
if _router_model_group is None:
|
||||
return None
|
||||
|
||||
return self._set_model_group_info(
|
||||
|
|
@ -13560,14 +13556,10 @@ class Router:
|
|||
alias_items = list(self.model_group_alias.items())
|
||||
|
||||
for model_alias, model_value in alias_items:
|
||||
if isinstance(model_value, str):
|
||||
_router_model_name: str = model_value
|
||||
elif isinstance(model_value, dict):
|
||||
_model_value = RouterModelGroupAliasItem(**model_value)
|
||||
if _model_value["hidden"] is True and model_name is None:
|
||||
continue
|
||||
_router_model_name = _model_value["model"]
|
||||
else:
|
||||
if isinstance(model_value, dict) and model_value.get("hidden") is True and model_name is None:
|
||||
continue
|
||||
_router_model_name = self.get_model_from_alias(model_alias)
|
||||
if _router_model_name is None:
|
||||
continue
|
||||
|
||||
if (alias_group := self.get_routing_group(_router_model_name)) is not None:
|
||||
|
|
@ -13831,9 +13823,7 @@ class Router:
|
|||
the wildcard route matching it when neither exists.
|
||||
|
||||
Unlike get_model_list, model_group's own model_group_alias entry is not
|
||||
followed. The router resolves an alias exactly once, so a group reached as
|
||||
an alias target is served by its own deployments, never by a second hop:
|
||||
in the chain X -> T -> U a request to X is served from T's deployments.
|
||||
followed. The caller has already resolved the complete alias chain.
|
||||
"""
|
||||
named: Final = [
|
||||
*self._get_all_deployments(model_name=model_group),
|
||||
|
|
|
|||
|
|
@ -103,15 +103,19 @@ def resolve_model_group_alias(model_group_alias: object, model: str) -> str | No
|
|||
entries: the map can come from a key or team row rather than from validated
|
||||
config, so a bad value must not raise mid-request.
|
||||
|
||||
Returns the target model group, or None when the map does not rewrite ``model``.
|
||||
Returns the final target model group, or None when no alias applies or the chain cycles.
|
||||
"""
|
||||
if not isinstance(model_group_alias, Mapping):
|
||||
if not isinstance(model_group_alias, Mapping) or model not in model_group_alias:
|
||||
return None
|
||||
entry: Final = model_group_alias.get(model)
|
||||
target: Final = entry.get("model") if isinstance(entry, Mapping) else entry
|
||||
if not isinstance(target, str) or not target:
|
||||
return None
|
||||
return target
|
||||
aliases: Final = _OBJECT_MAPPING_ADAPTER.validate_python(model_group_alias)
|
||||
current: str = model # rebind-ok: iterative traversal supports chains beyond Python's recursion limit
|
||||
for _ in range(len(aliases) + 1):
|
||||
entry = aliases.get(current)
|
||||
target = _OBJECT_MAPPING_ADAPTER.validate_python(entry).get("model") if isinstance(entry, Mapping) else entry
|
||||
if not isinstance(target, str) or not target:
|
||||
return current if current != model else None
|
||||
current = target
|
||||
return None
|
||||
|
||||
|
||||
def resolve_served_model(
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ def _await_rig(rig: ChainRig) -> None:
|
|||
tail_id: Final = _deployment_id(rig.tail_free)
|
||||
for candidate in rig.candidates:
|
||||
_await_served_by(candidate, rig.mid_free, paid_id)
|
||||
_await_served_by(candidate, rig.failing_free, paid_id)
|
||||
_await_served_by(candidate, rig.failing_free, _deployment_id(rig.failing_free))
|
||||
_await_served_by(candidate, rig.mid_paid, tail_id)
|
||||
_await_served_by(candidate, rig.unpriced, tail_id)
|
||||
|
||||
|
|
@ -331,11 +331,14 @@ def _chain_rig() -> Generator[ChainRig]:
|
|||
ghost_entry=f"chain-ghost-entry-{suffix}",
|
||||
failing_entry=f"chain-failing-entry-{suffix}",
|
||||
)
|
||||
free_middle: Final = f"{rig.entry}-middle"
|
||||
failing_middle: Final = f"{rig.failing_entry}-middle"
|
||||
aliases: Final[Mapping[str, JsonValue]] = {
|
||||
rig.entry: rig.mid_free,
|
||||
rig.hidden_entry: hidden(rig.mid_free),
|
||||
rig.shown_entry: {"model": rig.mid_free, "hidden": False},
|
||||
rig.null_entry: {"model": rig.mid_free, "hidden": None},
|
||||
rig.entry: free_middle,
|
||||
rig.hidden_entry: hidden(free_middle),
|
||||
rig.shown_entry: {"model": free_middle, "hidden": False},
|
||||
rig.null_entry: {"model": free_middle, "hidden": None},
|
||||
free_middle: rig.tail_free,
|
||||
rig.mid_free: rig.paid,
|
||||
rig.reverse_entry: rig.mid_paid,
|
||||
rig.mid_paid: rig.tail_free,
|
||||
|
|
@ -343,8 +346,8 @@ def _chain_rig() -> Generator[ChainRig]:
|
|||
rig.unpriced: rig.tail_free,
|
||||
rig.ghost_entry: rig.ghost_mid,
|
||||
rig.ghost_mid: rig.paid,
|
||||
rig.failing_entry: rig.failing_free,
|
||||
rig.failing_free: rig.paid,
|
||||
rig.failing_entry: failing_middle,
|
||||
failing_middle: rig.failing_free,
|
||||
}
|
||||
install_aliases(gateway, aliases)
|
||||
scenario.cleanups.callback(remove_aliases, gateway, frozenset(aliases))
|
||||
|
|
@ -524,46 +527,47 @@ def test_chain_middle_by_name_is_judged_by_its_own_alias(rig: ChainRig) -> None:
|
|||
assert row["model_group"] == rig.mid_free, row
|
||||
|
||||
|
||||
def test_reverse_chain_is_priced_by_the_middle_it_is_served_from(rig: ChainRig) -> None:
|
||||
def test_chain_through_a_paid_middle_is_priced_by_its_free_target(rig: ChainRig) -> None:
|
||||
free_marker: Final = "reverse-free-" + uuid.uuid4().hex
|
||||
billed_marker: Final = "reverse-billed-" + uuid.uuid4().hex
|
||||
entry_marker: Final = "reverse-entry-" + uuid.uuid4().hex
|
||||
with rig.gateway.scenario() as scenario:
|
||||
exhausted: Final = _exhausted_key(rig, scenario)
|
||||
for candidate in rig.candidates:
|
||||
_assert_refused_on_every_route(rig, candidate, exhausted, rig.reverse_entry)
|
||||
_assert_raw_chat_served(rig, candidate, rig.reverse_entry, exhausted, "reverse-chain")
|
||||
served: Final = fresh_chat(rig.gateway, rig.mid_paid, exhausted, free_marker)
|
||||
assert served.status_code == 200, served.text
|
||||
assert served.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict(served.headers)
|
||||
assert_free_row(landed_once(exhausted, free_marker), rig.mid_paid)
|
||||
budgeted: Final = scenario.key(max_budget=HEADROOM)
|
||||
billed: Final = fresh_chat(rig.gateway, rig.reverse_entry, budgeted, billed_marker)
|
||||
assert billed.status_code == 200, billed.text
|
||||
assert billed.headers["x-litellm-model-id"] == _deployment_id(rig.mid_paid), dict(billed.headers)
|
||||
row: Final = landed_once(budgeted, billed_marker)
|
||||
assert math.isclose(float(str(row["spend"])), PAID_REPLY_COST), row
|
||||
assert row["model_group"] == rig.reverse_entry, row
|
||||
entry_response: Final = fresh_chat(rig.gateway, rig.reverse_entry, exhausted, entry_marker)
|
||||
assert entry_response.status_code == 200, entry_response.text
|
||||
assert entry_response.headers["x-litellm-model-id"] == _deployment_id(rig.tail_free), dict(
|
||||
entry_response.headers
|
||||
)
|
||||
assert_free_row(landed_once(exhausted, entry_marker), rig.reverse_entry)
|
||||
|
||||
|
||||
def test_chain_through_a_cost_map_priced_middle_stays_budgeted(rig: ChainRig) -> None:
|
||||
def test_chain_through_a_cost_map_priced_middle_uses_the_free_targets_budget_waiver(rig: ChainRig) -> None:
|
||||
with rig.gateway.scenario() as scenario:
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
for candidate in rig.candidates:
|
||||
_assert_refused_on_every_route(rig, candidate, key, rig.unpriced_entry)
|
||||
_assert_raw_chat_served(rig, candidate, rig.unpriced_entry, key, "unpriced-middle")
|
||||
|
||||
|
||||
def _assert_ghost_chain(rig: ChainRig, candidate: Gateway, key: str) -> None:
|
||||
assert _group_rows(candidate, rig.ghost_entry) == ()
|
||||
assert rig.ghost_entry not in _listed_input_prices(candidate)
|
||||
_assert_price(
|
||||
_price_of(candidate, rig.ghost_entry), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",)
|
||||
)
|
||||
assert _listed_input_prices(candidate)[rig.ghost_entry] == PAID_INPUT_COST_PER_TOKEN
|
||||
refused: Final = fresh_chat(candidate, rig.ghost_entry, key, "ghost-exhausted-" + uuid.uuid4().hex)
|
||||
assert refused.status_code == BUDGET_EXCEEDED, refused.text
|
||||
unroutable: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex)
|
||||
assert unroutable.status_code == 400, unroutable.text
|
||||
assert "no healthy deployments" in unroutable.text, unroutable.text
|
||||
served: Final = fresh_chat(candidate, rig.ghost_entry, rig.gateway.key, "ghost-admin-" + uuid.uuid4().hex)
|
||||
assert served.status_code == 200, served.text
|
||||
assert served.headers["x-litellm-model-id"] == _deployment_id(rig.paid), dict(served.headers)
|
||||
for path in ("/health/liveliness", "/model/info", "/v1/models"):
|
||||
assert candidate.request("GET", path).status_code == 200, path
|
||||
|
||||
|
||||
def test_ghost_chain_reports_no_group_and_stays_unroutable(rig: ChainRig) -> None:
|
||||
def test_chain_through_an_alias_without_deployments_routes_to_its_paid_target(rig: ChainRig) -> None:
|
||||
with rig.gateway.scenario() as scenario:
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
for candidate in rig.candidates:
|
||||
|
|
@ -590,7 +594,7 @@ def test_provider_failure_behind_a_chain_entry_reaches_the_caller(rig: ChainRig)
|
|||
_assert_no_served_row(key, failed_marker)
|
||||
|
||||
|
||||
def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainRig) -> None:
|
||||
def test_chain_to_a_paid_target_does_not_inherit_a_free_wildcards_budget_waiver(rig: ChainRig) -> None:
|
||||
prefix: Final = "wc" + uuid.uuid4().hex[:8]
|
||||
middle: Final = f"{prefix}/gpt-4o-mini"
|
||||
other: Final = f"{prefix}/gpt-4o"
|
||||
|
|
@ -601,7 +605,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR
|
|||
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
|
||||
paid_id: Final = _deployment_id(rig.paid)
|
||||
for candidate in rig.candidates:
|
||||
_await_served_by(candidate, entry, route)
|
||||
_await_served_by(candidate, entry, paid_id)
|
||||
_await_served_by(candidate, middle, paid_id)
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
for candidate in rig.candidates:
|
||||
|
|
@ -610,7 +614,7 @@ def test_chain_through_a_free_wildcard_route_is_priced_by_that_route(rig: ChainR
|
|||
_assert_price(
|
||||
_price_of(candidate, middle), PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN, ("openai",)
|
||||
)
|
||||
_assert_served_by_route(candidate, entry, key, route, "wildcard-served")
|
||||
_assert_refused_on_every_route(rig, candidate, key, entry)
|
||||
_assert_refused(rig, candidate, key, middle, "wildcard-middle")
|
||||
|
||||
|
||||
|
|
@ -622,21 +626,21 @@ def _assert_served_by_route(candidate: Gateway, model: str, key: str, route: str
|
|||
assert_free_row(landed_once(key, marker), model)
|
||||
|
||||
|
||||
def test_chain_through_a_priced_wildcard_route_stays_budgeted(rig: ChainRig) -> None:
|
||||
def test_chain_to_a_free_target_takes_precedence_over_a_priced_wildcard_route(rig: ChainRig) -> None:
|
||||
prefix: Final = "pwc" + uuid.uuid4().hex[:8]
|
||||
middle: Final = f"{prefix}/gpt-4o-mini"
|
||||
entry: Final = "priced-wildcard-entry-" + uuid.uuid4().hex
|
||||
with rig.gateway.scenario() as scenario:
|
||||
route: Final = _wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN)
|
||||
_wildcard_route(rig, scenario, prefix, PAID_INPUT_COST_PER_TOKEN, PAID_OUTPUT_COST_PER_TOKEN)
|
||||
install_aliases(rig.gateway, {entry: middle, middle: rig.tail_free})
|
||||
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
|
||||
tail_id: Final = _deployment_id(rig.tail_free)
|
||||
for candidate in rig.candidates:
|
||||
_await_served_by(candidate, entry, route)
|
||||
_await_served_by(candidate, entry, tail_id)
|
||||
_await_served_by(candidate, middle, tail_id)
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
for candidate in rig.candidates:
|
||||
_assert_refused_on_every_route(rig, candidate, key, entry)
|
||||
_assert_raw_chat_served(rig, candidate, entry, key, "priced-wildcard-free-target")
|
||||
|
||||
|
||||
def _assert_hostile_group_queries_answered(rig: ChainRig, candidate: Gateway) -> None:
|
||||
|
|
@ -738,7 +742,7 @@ def test_duplicate_model_field_is_judged_by_its_last_value(rig: ChainRig) -> Non
|
|||
def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> None:
|
||||
marker: Final = "restricted-served-" + uuid.uuid4().hex
|
||||
with rig.gateway.scenario() as scenario:
|
||||
outsider: Final = scenario.key(models=[rig.tail_free, rig.paid], max_budget=BUDGET)
|
||||
outsider: Final = scenario.key(models=[rig.paid], max_budget=BUDGET)
|
||||
denied: Final = fresh_chat(rig.gateway, rig.entry, outsider, "restricted-denied-" + uuid.uuid4().hex)
|
||||
assert _refusal(denied) == (403, "key_model_access_denied"), denied.text
|
||||
insider: Final = scenario.key(models=[rig.entry, rig.paid], max_budget=BUDGET)
|
||||
|
|
@ -753,16 +757,21 @@ def test_key_restricted_to_the_chain_entry_keeps_the_waiver(rig: ChainRig) -> No
|
|||
def test_repointing_the_chain_middle_follows_the_served_deployment(rig: ChainRig) -> None:
|
||||
with rig.gateway.scenario() as scenario:
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
scenario.cleanups.callback(install_aliases, rig.gateway, {rig.entry: rig.mid_free, rig.mid_free: rig.paid})
|
||||
scenario.cleanups.callback(
|
||||
install_aliases, rig.gateway, {rig.entry: f"{rig.entry}-middle", rig.mid_free: rig.paid}
|
||||
)
|
||||
install_aliases(rig.gateway, {rig.entry: rig.mid_free})
|
||||
install_aliases(rig.gateway, {rig.mid_free: rig.tail_free})
|
||||
_settle_both(rig, rig.mid_free, key, 200)
|
||||
_settle_both(rig, rig.entry, key, 200)
|
||||
install_aliases(rig.gateway, {rig.mid_free: rig.paid})
|
||||
_settle_both(rig, rig.mid_free, key, BUDGET_EXCEEDED)
|
||||
_settle_both(rig, rig.entry, key, 200)
|
||||
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
|
||||
install_aliases(rig.gateway, {rig.entry: rig.paid})
|
||||
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
|
||||
install_aliases(rig.gateway, {rig.entry: rig.mid_free})
|
||||
_settle_both(rig, rig.entry, key, BUDGET_EXCEEDED)
|
||||
install_aliases(rig.gateway, {rig.entry: f"{rig.entry}-middle"})
|
||||
_settle_both(rig, rig.entry, key, 200)
|
||||
|
||||
|
||||
|
|
@ -942,8 +951,9 @@ def test_killing_one_worker_mid_burst_keeps_the_chain_entry_served(rig: ChainRig
|
|||
input_cost_per_token=0,
|
||||
output_cost_per_token=0,
|
||||
)
|
||||
install_aliases(rig.gateway, {entry: chaos_free, chaos_free: rig.paid})
|
||||
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, chaos_free}))
|
||||
middle: Final = f"{entry}-middle"
|
||||
install_aliases(rig.gateway, {entry: middle, middle: chaos_free})
|
||||
scenario.cleanups.callback(remove_aliases, rig.gateway, frozenset({entry, middle}))
|
||||
key: Final = _exhausted_key(rig, scenario)
|
||||
with owned_proxy_process(rig.gateway, tmp_path, {}, workers=2) as owned:
|
||||
candidate: Final = owned.gateway
|
||||
|
|
|
|||
|
|
@ -2020,10 +2020,6 @@ async def test_get_user_object_cache_miss_emits_exactly_one_postgres_get_user_ob
|
|||
def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_routes_to(
|
||||
monkeypatch: pytest.MonkeyPatch, entry_first: bool
|
||||
) -> None:
|
||||
"""chain-entry resolves one hop to local-free and is served by local-free's own free
|
||||
deployment, so an over-budget key is waived for it; local-free by name resolves to paid-gpt
|
||||
and stays enforced. local-free's alias is a hop the router never takes for chain-entry, and
|
||||
the per-name verdict cache must not let either name's verdict leak into the other's."""
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
|
|
@ -2048,7 +2044,7 @@ def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_ro
|
|||
],
|
||||
model_group_alias={"chain-entry": "local-free", "local-free": "paid-gpt"},
|
||||
)
|
||||
expected: Final = {"chain-entry": True, "local-free": False, "paid-gpt": False}
|
||||
expected: Final = {"chain-entry": False, "local-free": False, "paid-gpt": False}
|
||||
order: Final = ("chain-entry", "local-free", "paid-gpt") if entry_first else ("local-free", "paid-gpt", "chain-entry")
|
||||
|
||||
verdicts: Final = {name: is_model_cost_zero(model=name, llm_router=router) for name in order}
|
||||
|
|
|
|||
|
|
@ -581,12 +581,7 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
|
||||
assert is_model_cost_zero(model="free-model", llm_router=router) is False
|
||||
|
||||
def test_alias_chain_through_a_priced_group_enforces_budget(self):
|
||||
"""An alias to a group that is itself an alias key resolves one hop, like the router does.
|
||||
|
||||
The router serves ``chain-smart`` with the real ``chain-legacy`` deployment, which is priced,
|
||||
so following the second hop to the free group would waive the budget for a paid call.
|
||||
"""
|
||||
def test_alias_chain_through_a_priced_group_to_a_free_target_bypasses_budget(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -613,18 +608,11 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={"chain-smart": "chain-legacy", "chain-legacy": "free-model"},
|
||||
)
|
||||
|
||||
assert is_model_cost_zero(model="chain-smart", llm_router=router) is False
|
||||
assert _served_model(router, "chain-smart") == "ollama/llama2"
|
||||
assert is_model_cost_zero(model="chain-smart", llm_router=router) is True
|
||||
|
||||
@pytest.mark.parametrize("hidden", [False, True], ids=["plain_alias", "hidden_alias"])
|
||||
def test_alias_to_an_unpriced_group_that_is_also_an_alias_enforces_budget(self, hidden: bool):
|
||||
"""An alias is judged by the group it routes to, never by where that group's own alias points.
|
||||
|
||||
The router serves ``chain-entry`` from the real ``chain-middle`` deployment, whose zero price
|
||||
comes from the cost map alone. ``chain-middle`` is also an alias key to an explicitly free
|
||||
group, a second hop the router never takes for ``chain-entry``, so that group must not lend
|
||||
it the bypass.
|
||||
"""
|
||||
|
||||
def test_alias_chain_through_an_unpriced_group_uses_the_explicitly_free_target(self, hidden: bool):
|
||||
def alias(target: str) -> str | dict[str, str | bool]:
|
||||
return {"model": target, "hidden": True} if hidden else target
|
||||
|
||||
|
|
@ -633,19 +621,11 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={"chain-entry": alias("chain-middle"), "chain-middle": alias("free-model")},
|
||||
)
|
||||
|
||||
assert _served_model(router, "chain-entry") == UNPRICED_ZERO_COST_MODEL
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
|
||||
assert is_model_cost_zero(model="chain-middle", llm_router=router) is True, (
|
||||
"asked by its own name, chain-middle routes to the explicitly free group"
|
||||
)
|
||||
assert _served_model(router, "chain-entry") == "gpt-3.5-turbo"
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
|
||||
assert is_model_cost_zero(model="chain-middle", llm_router=router) is True
|
||||
|
||||
def test_alias_to_a_free_group_that_is_also_an_alias_to_a_ptu_group_bypasses_budget(self):
|
||||
"""A PTU group one alias hop past the group that serves the request does not enforce the budget.
|
||||
|
||||
The router serves ``chain-entry`` from the real, explicitly free ``chain-middle`` deployment.
|
||||
``chain-middle`` is also an alias key to a PTU-priced group, which only a request for
|
||||
``chain-middle`` itself routes to.
|
||||
"""
|
||||
def test_alias_chain_through_a_free_group_to_a_ptu_target_enforces_budget(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
_explicitly_free("chain-middle"),
|
||||
|
|
@ -654,19 +634,11 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={"chain-entry": "chain-middle", "chain-middle": "ptu-model"},
|
||||
)
|
||||
|
||||
assert _served_model(router, "chain-entry") == "gpt-3.5-turbo"
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
|
||||
assert is_model_cost_zero(model="chain-middle", llm_router=router) is False, (
|
||||
"asked by its own name, chain-middle routes to the PTU group"
|
||||
)
|
||||
assert _served_model(router, "chain-entry") == "azure/ptu-deployment"
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
|
||||
assert is_model_cost_zero(model="chain-middle", llm_router=router) is False
|
||||
|
||||
def test_alias_chain_served_by_a_paid_wildcard_route_enforces_budget(self):
|
||||
"""An alias whose target is only an alias key is never judged by that second alias's free group.
|
||||
|
||||
``chain-entry`` resolves one hop to ``gpt-4o-mini``, which is no deployment's name, so the
|
||||
router serves it from the paid wildcard route. The free group ``gpt-4o-mini`` is aliased
|
||||
to is only reached by a request for ``gpt-4o-mini`` itself.
|
||||
"""
|
||||
def test_alias_chain_to_a_free_target_takes_precedence_over_a_paid_wildcard(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
_explicitly_free("free-model", model="ollama/llama2"),
|
||||
|
|
@ -675,17 +647,11 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={"chain-entry": "gpt-4o-mini", "gpt-4o-mini": "free-model"},
|
||||
)
|
||||
|
||||
assert _served_model(router, "chain-entry") == "openai/gpt-4o-mini"
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is False
|
||||
assert _served_model(router, "chain-entry") == "ollama/llama2"
|
||||
assert is_model_cost_zero(model="chain-entry", llm_router=router) is True
|
||||
|
||||
@pytest.mark.parametrize("alias_name", ["openai/smart", "smart"], ids=["alias_on_pattern", "alias_off_pattern"])
|
||||
def test_alias_chain_served_by_an_explicitly_priced_wildcard_route_enforces_budget(self, alias_name: str):
|
||||
"""A chain the router serves from a priced wildcard route is budgeted at that route's price.
|
||||
|
||||
The group's price reads $0 through the second alias to the free group, and the wildcard
|
||||
route's cost-map entry carries explicit prices, so only their sign tells that the served
|
||||
deployment is not free.
|
||||
"""
|
||||
def test_alias_chain_to_a_free_target_takes_precedence_over_an_explicitly_priced_wildcard(self, alias_name: str):
|
||||
router = Router(
|
||||
model_list=[
|
||||
_explicitly_free("free-model", model="ollama/llama2"),
|
||||
|
|
@ -703,11 +669,10 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={alias_name: "openai/gpt-4o-mini", "openai/gpt-4o-mini": "free-model"},
|
||||
)
|
||||
|
||||
assert _served_model(router, alias_name) == "openai/gpt-4o-mini"
|
||||
assert is_model_cost_zero(model=alias_name, llm_router=router) is False
|
||||
assert _served_model(router, alias_name) == "ollama/llama2"
|
||||
assert is_model_cost_zero(model=alias_name, llm_router=router) is True
|
||||
|
||||
def test_alias_shadowing_a_free_group_is_judged_by_its_unpriced_target_through_an_alias_chain(self):
|
||||
"""A shadowing alias stays enforced when its unpriced target is itself an alias key to a free group."""
|
||||
def test_alias_shadowing_a_free_group_is_judged_by_the_final_target_through_an_unpriced_group(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
_explicitly_free("shadowed-free"),
|
||||
|
|
@ -717,8 +682,8 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
model_group_alias={"shadowed-free": "chain-middle", "chain-middle": "free-model"},
|
||||
)
|
||||
|
||||
assert _served_model(router, "shadowed-free") == UNPRICED_ZERO_COST_MODEL
|
||||
assert is_model_cost_zero(model="shadowed-free", llm_router=router) is False
|
||||
assert _served_model(router, "shadowed-free") == "gpt-3.5-turbo"
|
||||
assert is_model_cost_zero(model="shadowed-free", llm_router=router) is True
|
||||
|
||||
@pytest.mark.parametrize("alias_name", ["ollama/fast", "fast"], ids=["alias_on_pattern", "alias_off_pattern"])
|
||||
def test_alias_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self, alias_name: str):
|
||||
|
|
@ -732,23 +697,15 @@ class TestUnmappedModelBudgetEnforcement:
|
|||
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is True
|
||||
assert is_model_cost_zero(model=alias_name, llm_router=router) is True
|
||||
|
||||
def test_alias_chain_to_a_name_served_by_an_explicitly_free_wildcard_route_bypasses_budget(self):
|
||||
"""An alias to an alias key no deployment is named after reads the wildcard route serving it.
|
||||
|
||||
``ollama/fast`` resolves one hop to ``ollama/llama3``, which is only an alias key, so the
|
||||
router serves it from the explicitly free wildcard route. The group ``ollama/llama3`` is
|
||||
aliased to is only reached by a request for ``ollama/llama3`` itself.
|
||||
"""
|
||||
def test_alias_chain_to_an_unpriced_target_does_not_inherit_a_free_wildcards_budget_waiver(self):
|
||||
router = Router(
|
||||
model_list=[_explicitly_free_ollama_wildcard(), _free_by_cost_map_only("unpriced-model")],
|
||||
model_group_alias={"ollama/fast": "ollama/llama3", "ollama/llama3": "unpriced-model"},
|
||||
)
|
||||
|
||||
assert _served_model(router, "ollama/fast") == "ollama/llama3"
|
||||
assert is_model_cost_zero(model="ollama/fast", llm_router=router) is True
|
||||
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False, (
|
||||
"asked by its own name, ollama/llama3 routes to the unpriced group"
|
||||
)
|
||||
assert _served_model(router, "ollama/fast") == UNPRICED_ZERO_COST_MODEL
|
||||
assert is_model_cost_zero(model="ollama/fast", llm_router=router) is False
|
||||
assert is_model_cost_zero(model="ollama/llama3", llm_router=router) is False
|
||||
|
||||
def test_handles_router_without_zero_cost_cache_attribute(self):
|
||||
"""Tolerate router-like objects (e.g. ``MagicMock`` stand-ins) that
|
||||
|
|
|
|||
|
|
@ -9567,6 +9567,8 @@ class TestPerRequestModelGroupAlias:
|
|||
({"group-a": {"hidden": True}}, None),
|
||||
({}, None),
|
||||
(None, None),
|
||||
({"group-a": "middle-alias", "middle-alias": {"model": "group-b", "hidden": True}}, "group-b"),
|
||||
({"group-a": "middle-alias", "middle-alias": "group-a"}, None),
|
||||
],
|
||||
)
|
||||
async def test_resolves_alias_for_the_requested_model_group(self, alias_map, expected):
|
||||
|
|
@ -9580,13 +9582,20 @@ class TestPerRequestModelGroupAlias:
|
|||
assert resolved == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_target_outside_the_key_allowlist_is_rejected(self):
|
||||
@pytest.mark.parametrize(
|
||||
"aliases",
|
||||
(
|
||||
{"group-a": "group-b"},
|
||||
{"group-a": "middle-alias", "middle-alias": "group-b"},
|
||||
),
|
||||
)
|
||||
async def test_alias_target_outside_the_key_allowlist_is_rejected(self, aliases: Mapping[str, str]) -> None:
|
||||
"""Access was authorized against the requested group, so a rewrite that
|
||||
the key could not have requested directly must not be served."""
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _resolve_per_request_model_group_alias(
|
||||
requested_model="group-a",
|
||||
router_settings={"model_group_alias": {"group-a": "group-b"}},
|
||||
router_settings={"model_group_alias": aliases},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash", models=["group-a"]),
|
||||
llm_router=self._router(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3018,7 +3018,6 @@ THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=Non
|
|||
OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None)
|
||||
FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1")
|
||||
SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None)
|
||||
ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1")
|
||||
PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None)
|
||||
EVERY_DEPLOYMENT: Final = (
|
||||
"refused-1",
|
||||
|
|
@ -3142,8 +3141,8 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD)
|
|||
("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", ALL_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", ALL_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)),
|
||||
),
|
||||
|
|
|
|||
|
|
@ -565,11 +565,27 @@ class TestResolveModelGroupAlias:
|
|||
({"group-a": 5}, None),
|
||||
({"group-a": None}, None),
|
||||
({"group-a": ""}, None),
|
||||
({"group-a": "group-b", "group-b": "group-c"}, "group-c"),
|
||||
(
|
||||
{"group-a": {"model": "group-b"}, "group-b": {"model": "group-c", "hidden": True}},
|
||||
"group-c",
|
||||
),
|
||||
({"group-a": "group-b", "group-b": {"hidden": True}}, "group-b"),
|
||||
({"group-a": "group-a"}, None),
|
||||
({"group-a": "group-b", "group-b": "group-a"}, None),
|
||||
({"group-a": "group-b", "group-b": "group-c", "group-c": "group-b"}, None),
|
||||
],
|
||||
)
|
||||
def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(self, alias_map, expected):
|
||||
def test_resolves_both_entry_shapes_and_tolerates_malformed_entries(
|
||||
self, alias_map: object, expected: str | None
|
||||
) -> None:
|
||||
assert resolve_model_group_alias(alias_map, "group-a") == expected
|
||||
|
||||
def test_resolves_alias_chains_longer_than_the_python_recursion_limit(self) -> None:
|
||||
aliases: Final = {f"group-{index}": f"group-{index + 1}" for index in range(1100)}
|
||||
|
||||
assert resolve_model_group_alias(aliases, "group-0") == "group-1100"
|
||||
|
||||
def test_router_alias_resolution_uses_the_shared_helper(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
|
|
@ -593,6 +609,7 @@ class TestResolveModelGroupAlias:
|
|||
pytest.param("a", {}, ["a", "b"], {}, "a", id="non-mapping-key-aliases-ignored"),
|
||||
pytest.param("a", {}, {}, {"a": {"model": "x"}}, "x", id="router-item-entry"),
|
||||
pytest.param("a", {}, {}, {}, "a", id="no-alias"),
|
||||
pytest.param("a", {"a": "b"}, {"b": "c"}, {"c": "d", "d": "e"}, "e", id="router-alias-chain"),
|
||||
],
|
||||
)
|
||||
def test_resolves_served_model_aliases_in_order(
|
||||
|
|
|
|||
|
|
@ -2742,9 +2742,6 @@ def _assert_free(info: ModelGroupInfo | None, provider: str) -> None:
|
|||
|
||||
|
||||
def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to():
|
||||
"""chain-entry resolves one hop to local-free and is served by local-free's own
|
||||
deployment, so its price is that deployment's; local-free's own alias to gpt-priced
|
||||
is a hop the router takes only for a request to local-free by name."""
|
||||
router = Router(
|
||||
model_list=[
|
||||
_free_ollama_deployment("local-free"),
|
||||
|
|
@ -2753,14 +2750,11 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_group_it_routes_to(
|
|||
model_group_alias={"chain-entry": "local-free", "local-free": "gpt-priced"},
|
||||
)
|
||||
|
||||
_assert_free(router.get_model_group_info(model_group="chain-entry"), "ollama")
|
||||
_assert_priced(router.get_model_group_info(model_group="chain-entry"), "openai")
|
||||
_assert_priced(router.get_model_group_info(model_group="local-free"), "openai")
|
||||
|
||||
|
||||
def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serving_it():
|
||||
"""When the routed group has no deployment of its own, the wildcard route matching it
|
||||
serves the request, so the price is the wildcard's and never the routed group's own alias
|
||||
target's."""
|
||||
router = Router(
|
||||
model_list=[
|
||||
_paid_openai_deployment("openai/*", "openai/*"),
|
||||
|
|
@ -2769,7 +2763,7 @@ def test_get_model_group_info_prices_an_alias_chain_from_the_wildcard_route_serv
|
|||
model_group_alias={"wildcard-entry": "openai/gpt-4o", "openai/gpt-4o": "local-free"},
|
||||
)
|
||||
|
||||
_assert_priced(router.get_model_group_info(model_group="wildcard-entry"), "openai")
|
||||
_assert_free(router.get_model_group_info(model_group="wildcard-entry"), "ollama")
|
||||
_assert_free(router.get_model_group_info(model_group="openai/gpt-4o"), "ollama")
|
||||
|
||||
|
||||
|
|
@ -2778,9 +2772,6 @@ def _served_models(deployments: list[DeploymentTypedDict] | None) -> list[str]:
|
|||
|
||||
|
||||
def test_get_model_list_of_routed_group_reads_the_groups_own_deployments_only():
|
||||
"""The router resolves an alias once, so a group reached as an alias target is served by
|
||||
its own deployments. get_model_list composes the group's own alias target too, the hop a
|
||||
request to that group by name takes."""
|
||||
router = Router(
|
||||
model_list=[
|
||||
_free_ollama_deployment("local-free"),
|
||||
|
|
@ -27560,6 +27551,36 @@ def test_router_get_model_list_from_model_alias() -> None:
|
|||
assert len(model_alias_list) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden", (False, True))
|
||||
@pytest.mark.parametrize("use_async", (False, True))
|
||||
async def test_chained_model_group_alias_selects_the_final_deployment(hidden: bool, use_async: bool) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "concrete-model",
|
||||
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-fake"},
|
||||
"model_info": {"id": "concrete-deployment"},
|
||||
}
|
||||
],
|
||||
model_group_alias={
|
||||
"public-alias": "middle-alias",
|
||||
"middle-alias": {"model": "concrete-model", "hidden": hidden},
|
||||
},
|
||||
)
|
||||
|
||||
deployment: Final = (
|
||||
await router.async_get_available_deployment(model="public-alias", messages=[], request_kwargs={})
|
||||
if use_async
|
||||
else router.get_available_deployment(model="public-alias", messages=[])
|
||||
)
|
||||
|
||||
assert deployment["model_info"]["id"] == "concrete-deployment"
|
||||
assert router.routable_model_group("public-alias") == "concrete-model"
|
||||
assert _served_models(router.get_model_list(model_name="public-alias")) == ["openai/gpt-6.1-sol"]
|
||||
assert "public-alias" in router.get_model_names()
|
||||
assert ("middle-alias" in router.get_model_names()) is not hidden
|
||||
|
||||
|
||||
def test_router_dynamic_credentials() -> None:
|
||||
"""
|
||||
Assert model id for dynamic api key 1 != model id for dynamic api key 2
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue