diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 45a15d0d1f1..7fbbaf84422 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -345,16 +345,22 @@ def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None: # The mirrored per-token pricing fields plus the three remaining fields # Router._inherit_builtin_cache_pricing back-fills from the public cost map. An unset field is # what that back-fill targets, so a field left out here is one a PTU deployment still bills. -# tiered_pricing is the one mirrored field that is a list, not a rate, so it is dropped from a -# PTU deployment (see _PTU_CLEARED_PRICING_FIELDS) rather than stored as zero. +# tiered_pricing is the one mirrored field that is a table of ranges, not a rate, so it is stored +# empty (see _PTU_EMPTIED_PRICING_FIELDS): its tiers outrank the zeros written beside them, so +# dropping it would leave the cost map's tiers billing the traffic the reserved capacity covers. _PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in SPECIAL_MODEL_INFO_PARAMS if f != "tiered_pricing") + ( "cache_creation_input_token_cost_above_1hr", "cache_creation_input_token_cost_above_200k_tokens", "cache_read_input_token_cost_above_200k_tokens", ) -_PTU_CLEARED_PRICING_FIELDS: Final = frozenset({"tiered_pricing"}) -_PTU_ZEROED_PRICING: Final[Mapping[str, float]] = MappingProxyType(dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0)) -_NO_PRICING_OVERRIDE: Final[Mapping[str, float]] = MappingProxyType({}) +_PTU_EMPTIED_PRICING_FIELDS: Final = frozenset({"tiered_pricing"}) +_PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()]]] = MappingProxyType( + { + **dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0), + **dict.fromkeys(_PTU_EMPTIED_PRICING_FIELDS, ()), + } +) +_NO_PRICING_OVERRIDE: Final[Mapping[str, float | tuple[()]]] = MappingProxyType({}) _EMPTY_MODEL_INFO: Final[Mapping[str, object]] = _NO_PRICING_OVERRIDE # Rate fields only. CustomPricingLiteLLMParams also carries settings that are not charges # (an embedding's output_vector_size, the regional uplift multipliers), and zeroing one of @@ -367,6 +373,8 @@ def _is_nonzero_price(value: object) -> bool: def _is_zero_price(value: object) -> bool: + if isinstance(value, (list, tuple)): + return not value return isinstance(value, (int, float)) and not isinstance(value, bool) and value == 0 @@ -384,7 +392,7 @@ def _raise_if_ptu_deployment_is_priced(*, model_info: Mapping[str, object], supp priced: Final = tuple( sorted( tuple(field for field in _CUSTOM_PRICING_FIELDS if _is_nonzero_price(supplied.get(field))) - + tuple(field for field in _PTU_CLEARED_PRICING_FIELDS if supplied.get(field)) + + tuple(field for field in _PTU_EMPTIED_PRICING_FIELDS if supplied.get(field)) ) ) if not priced: @@ -403,7 +411,7 @@ def _ptu_zeroed_pricing( model_info: Mapping[str, object], litellm_params: Mapping[str, object], supplied: Mapping[str, object], -) -> Mapping[str, float]: +) -> Mapping[str, float | tuple[()]]: """The pricing a PTU deployment must carry, empty unless one is being stored. Reserved capacity is already billed by the flat cost the rollup writes, so charging the @@ -440,7 +448,7 @@ def _ptu_pricing_delta( model_info: Mapping[str, object], litellm_params: Mapping[str, object], patch: updateDeployment, -) -> tuple[Mapping[str, float], frozenset[str]]: +) -> tuple[Mapping[str, float | tuple[()]], frozenset[str]]: """The pricing a patch must write into both blobs, and the pricing it must drop from them. A patch that takes the deployment off PTU takes the zeroed pricing with it, since the zeros @@ -456,13 +464,13 @@ def _ptu_pricing_delta( supplied: Final = patch.litellm_params.model_dump(exclude_none=True) if patch.litellm_params else _EMPTY_MODEL_INFO zeroed: Final = _ptu_zeroed_pricing(model_info=model_info, litellm_params=litellm_params, supplied=supplied) if zeroed: - return zeroed, _PTU_CLEARED_PRICING_FIELDS + return zeroed, frozenset() was_ptu: Final = any(stored_model_info.get(field) is not None for field in _PTU_PRICED_PAIR) if not was_ptu or not _explicitly_cleared_ptu_fields(patch.model_info) & _PTU_PRICED_PAIR: return _NO_PRICING_OVERRIDE, frozenset() return _NO_PRICING_OVERRIDE, frozenset( field - for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS) + for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS, _PTU_EMPTIED_PRICING_FIELDS) if _is_zero_price(model_info.get(field)) or _is_zero_price(litellm_params.get(field)) ) @@ -474,13 +482,16 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: override: Final = _ptu_zeroed_pricing(model_info=model_info, litellm_params=litellm_params, supplied=litellm_params) if not override: return model_params - cleared: Final = dict.fromkeys(_PTU_CLEARED_PRICING_FIELDS, None) - pricing_update: Final = MappingProxyType(dict(override, **cleared)) + # model_copy validates nothing, so the emptied tier table has to arrive as the list the field + # declares or Pydantic warns on every later dump of it + stored: Final = MappingProxyType( + {key: [] if isinstance(value, tuple) else value for key, value in override.items()} + ) return model_params.model_copy( update=MappingProxyType( { - "litellm_params": model_params.litellm_params.model_copy(update=pricing_update), - "model_info": model_params.model_info.model_copy(update=pricing_update), + "litellm_params": model_params.litellm_params.model_copy(update=stored), + "model_info": model_params.model_info.model_copy(update=stored), } ) ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index 5af880038eb..d3aec8010f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -15,6 +15,7 @@ from litellm.proxy._types import ( ReconcileOutcome, UserAPIKeyAuth, ) +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.proxy.auth.auth_checks import _is_model_cost_zero from litellm.proxy.management_endpoints.model_management_endpoints import ( _PTU_ZEROED_PRICING_FIELDS, @@ -37,6 +38,7 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, ) +from litellm.types.utils import Usage def test_model_info_accepts_valid_ptu_fields(): @@ -762,7 +764,10 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert self._zeroed(model_info={"ptu_count": 15}) == {} def test_every_field_the_cost_map_could_fill_is_zeroed(self): - assert self._zeroed(model_info=self.PTU) == dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0) + assert self._zeroed(model_info=self.PTU) == { + **dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0), + "tiered_pricing": (), + } def test_nothing_is_zeroed_while_the_feature_is_disabled(self, monkeypatch): monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False) @@ -784,9 +789,10 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert exc.value.status_code == 400 assert "tiered_pricing" in str(exc.value.detail) - def test_tiered_pricing_already_on_the_row_is_cleared_not_zeroed(self): - """tiered_pricing is a list, so the zero the other fields store would not even validate. - Left in place it would keep billing per token at the tier rates.""" + def test_tiered_pricing_already_on_the_row_is_emptied_not_zeroed(self): + """tiered_pricing is a table of ranges, so the zero the other fields store would not even + validate. Dropping it instead would fall back to the cost map's tiers, whose rates outrank + the zeros written beside them, so it is stored empty.""" tiers = [{"range": [0, 128000], "input_cost_per_token": 3e-06}] priced = _ptu_priced_deployment( Deployment( @@ -801,8 +807,8 @@ class TestPtuDeploymentsAreNotBilledPerToken: ), ) ) - assert priced.litellm_params.tiered_pricing is None - assert priced.model_info.tiered_pricing is None + assert priced.litellm_params.tiered_pricing == [] + assert priced.model_info.tiered_pricing == [] written = update_db_model( db_model=Deployment( @@ -821,7 +827,7 @@ class TestPtuDeploymentsAreNotBilledPerToken: ) for blob in ("model_info", "litellm_params"): stored = json.loads(written[blob]) - assert "tiered_pricing" not in stored, blob + assert stored["tiered_pricing"] == [], blob assert stored["input_cost_per_token"] == 0, blob def test_a_price_the_caller_supplies_as_zero_is_accepted(self): @@ -957,6 +963,32 @@ class TestPtuDeploymentsAreNotBilledPerToken: charged = {k: v for k, v in registered.items() if "cost" in k and k != "cost_per_ptu_per_hour" and v} assert charged == {} + def test_the_cost_map_tiers_contribute_no_price_to_a_priced_ptu_deployment(self): + """A tier table outranks the zeroed flat rates wherever cost is read, so leaving the + deployment's own table unset bills the reserved capacity's traffic at the map's tiers.""" + priced = _ptu_priced_deployment( + Deployment( + model_name="ptu-deployment", + litellm_params=LiteLLM_Params(model="dashscope/qwen-flash", api_key="fake-key"), + model_info=ModelInfo( + id="dep-ptu", + team_id="team-1", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ), + ) + ) + router = Router(model_list=[priced.to_json(exclude_none=True)]) + registered = router.get_deployment_model_info(model_id="dep-ptu", model_name="dashscope/qwen-flash") + assert registered is not None + assert registered["tiered_pricing"] == [] + assert generic_cost_per_token( + model="dashscope/qwen-flash", + usage=Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100), + custom_llm_provider="dashscope", + model_info=registered, + ) == (0.0, 0.0) + def test_the_zeroed_pricing_does_not_waive_budget_enforcement(self): """A zero price otherwise tells auth the model is free and skips every budget check.""" priced = _ptu_priced_deployment( @@ -1146,9 +1178,9 @@ class TestPtuDeploymentsAreNotBilledPerToken: written = add_team_model_to_db.call_args.kwargs["model_params"] assert all(getattr(written.model_info, field, None) == 0 for field in SPECIAL_MODEL_INFO_PARAMS if field != "tiered_pricing") - assert written.model_info.tiered_pricing is None + assert written.model_info.tiered_pricing == [] assert all(written.litellm_params.get(field) == 0 for field in _PTU_ZEROED_PRICING_FIELDS) - assert written.litellm_params.tiered_pricing is None + assert written.litellm_params.tiered_pricing == [] @pytest.mark.asyncio async def test_model_new_refuses_a_priced_ptu_deployment(self):