mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(ptu): keep a declared split as the access rule and check a config split at boot with the flag off
6fb24d1d read ptu_shares through the pricing terms so a termless map reserved
nothing, but with the flag off the router skips ptu_config_error for config.yaml
entries, so such an entry loaded and served every team, against the rule that a
declared split restricts access whether or not cost attribution is on.
_deployment_shares reads the declared map again, and the router now raises
ptu_config_error for a config-sourced entry declaring ptu_shares regardless of
the flag, so a malformed split is refused at boot instead of being honoured or
ignored silently.
This commit is contained in:
parent
6fb24d1dad
commit
e4b33bc8a9
4 changed files with 27 additions and 17 deletions
|
|
@ -8957,7 +8957,8 @@ class Router:
|
|||
ptu_error: Final = (
|
||||
(ptu_config_error(_model_info, model_name=_model_name) or identity_error) if config_sourced else None
|
||||
)
|
||||
if ptu_error is not None and is_ptu_cost_attribution_enabled():
|
||||
declares_split: Final = _model_info.get("ptu_shares") is not None
|
||||
if ptu_error is not None and (declares_split or is_ptu_cost_attribution_enabled()):
|
||||
raise ValueError(ptu_error)
|
||||
access_windows_error: Final = access_windows_config_error(_model_info, model_name=_model_name)
|
||||
if access_windows_error is not None:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_model_info_mapping, ptu_terms
|
||||
from litellm.litellm_core_utils.ptu_pricing import is_model_info_mapping, parsed_ptu_shares, ptu_terms
|
||||
from litellm.llms.azure.ptu_capacity import PTUCapacity, deployment_ptu_capacity, is_azure_deployment
|
||||
from litellm.router_utils.common_utils import team_may_use_deployment
|
||||
|
||||
|
|
@ -36,17 +36,15 @@ class PTUShareFilterResult(Generic[_DeploymentT]):
|
|||
|
||||
|
||||
def _deployment_shares(deployment: Mapping[str, object]) -> Mapping[str, int] | None:
|
||||
"""The teams a deployment's capacity is split across, else None.
|
||||
"""The teams a deployment is declared split across, else None.
|
||||
|
||||
Only a map registration would accept counts, read through the same terms flat cost accrues
|
||||
under, so a row that reached the table without its count, rate and start reserves nothing
|
||||
instead of refusing every other team while charging nobody.
|
||||
The declared map is the access rule on its own; the pricing terms are read separately, so a
|
||||
split whose terms are missing still serves only the teams it names rather than everyone.
|
||||
"""
|
||||
model_info: Final = deployment.get("model_info")
|
||||
if not is_model_info_mapping(model_info) or model_info.get("ptu_shares") is None:
|
||||
if not is_model_info_mapping(model_info):
|
||||
return None
|
||||
terms: Final = ptu_terms(model_info)
|
||||
return None if terms is None else terms.shares
|
||||
return parsed_ptu_shares(model_info.get("ptu_shares"))
|
||||
|
||||
|
||||
def filter_ptu_shared_deployments(
|
||||
|
|
|
|||
|
|
@ -90,14 +90,15 @@ _TERMLESS_SHARED: Final = {
|
|||
}
|
||||
|
||||
|
||||
def test_a_share_map_registration_would_refuse_reserves_nothing():
|
||||
"""Registration refuses ``ptu_shares`` without the count, rate and start they split, and a map
|
||||
that does not add up to the count, so a row carrying either shape holds nobody's capacity:
|
||||
every team is served from it and no ceiling applies, the same as before the map was read."""
|
||||
short: Final = _shared(shares={"team-a": 30}, ptu_count=50)
|
||||
for deployment in (_TERMLESS_SHARED, short):
|
||||
assert filter_ptu_shared_deployments([deployment], "team-z").withheld is False
|
||||
assert _unaliased_ceiling([deployment], "team-a", "gpt-4.1-ptu") is None
|
||||
def test_a_declared_split_is_the_access_rule_even_without_pricing_terms():
|
||||
"""A map that reached the router without the count, rate and start registration now demands
|
||||
still names who the deployment is for, so it is served to those teams alone and sized from
|
||||
their shares; only the flat cost, which needs the terms, goes unattributed."""
|
||||
assert filter_ptu_shared_deployments([_TERMLESS_SHARED], "team-z").withheld is True
|
||||
assert filter_ptu_shared_deployments([_TERMLESS_SHARED], "team-a").withheld is False
|
||||
ceiling: Final = _unaliased_ceiling([_TERMLESS_SHARED], "team-a", "gpt-4.1-ptu")
|
||||
assert ceiling is not None
|
||||
assert ceiling.tpm_limit == 30 * _GPT41.input_tpm_per_ptu
|
||||
|
||||
|
||||
def test_a_team_is_served_from_its_shared_deployment_first_and_never_from_another_teams_reservation():
|
||||
|
|
|
|||
|
|
@ -18845,6 +18845,16 @@ def test_a_shared_ptu_deployment_whose_shares_do_not_add_up_is_refused_at_regist
|
|||
Router(model_list=model_list)
|
||||
|
||||
|
||||
def test_a_config_entry_declaring_shares_without_terms_is_refused_even_while_the_feature_is_off(monkeypatch):
|
||||
"""The split is enforced with the flag off, so its shape is checked with the flag off too: a
|
||||
malformed one is refused at boot instead of being honoured or ignored without a word."""
|
||||
monkeypatch.delenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", raising=False)
|
||||
model_list = _shared_ptu_model_list()[:1]
|
||||
model_list[0]["model_info"] = {"id": "shared-deployment", "ptu_shares": {"team-a": 30, "team-b": 20}}
|
||||
with pytest.raises(ValueError, match=r"ptu_count and cost_per_ptu_per_hour are required when ptu_shares is set"):
|
||||
Router(model_list=model_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"verdict",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue