diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index 2e73719cf52..021210d9175 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -124,6 +124,14 @@ def ptu_identity_error( return None +PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to") + + +def declares_ptu(model_info: Mapping[str, object]) -> bool: + """Whether any PTU field is set here, including one too malformed to charge.""" + return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS) + + def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None: """Why this PTU configuration cannot be honoured, else None. diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b003daa9d79..217fc61a56c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -27,6 +27,7 @@ from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.litellm_core_utils.ptu_pricing import ( CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, + PTU_MODEL_INFO_FIELDS, PTU_ZEROED_PRICING_FIELDS, PTU_ZEROED_TABLE_FIELDS, SEARCH_CONTEXT_SIZES, @@ -247,7 +248,6 @@ def _raise_on_strategy_router_write_violation( ) -_PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to") _PTU_PRICED_PAIR: Final = frozenset({"ptu_count", "cost_per_ptu_per_hour"}) @@ -261,7 +261,7 @@ def _explicitly_cleared_ptu_fields(model_info: ModelInfo | None) -> frozenset[st return frozenset() return frozenset( field - for field in _PTU_MODEL_INFO_FIELDS + for field in PTU_MODEL_INFO_FIELDS if field in model_info.model_fields_set and getattr(model_info, field) is None ) @@ -294,7 +294,7 @@ def _raise_if_ptu_cost_attribution_disabled(incoming_model_info: Mapping[str, ob """ if is_ptu_cost_attribution_enabled(): return - supplied: Final = tuple(field for field in _PTU_MODEL_INFO_FIELDS if incoming_model_info.get(field) is not None) + supplied: Final = tuple(field for field in PTU_MODEL_INFO_FIELDS if incoming_model_info.get(field) is not None) if not supplied: return raise HTTPException( diff --git a/litellm/router.py b/litellm/router.py index 7dedbe851d7..045fd32847c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -66,6 +66,8 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.ptu_pricing import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + declares_ptu, is_ptu_cost_attribution_enabled, ptu_config_error, ptu_identity_error, @@ -8234,6 +8236,21 @@ class Router: ) duplicate_ids: Final = frozenset(model_id for model_id in declared_ids if declared_ids.count(model_id) > 1) + ptu_declared: Final = tuple( + str(entry.get("model_name")) + for entry in original_model_list + if isinstance(entry.get("model_info"), dict) + and entry["model_info"].get("db_model") is not True + and declares_ptu(entry["model_info"]) + ) + if ptu_declared and not is_ptu_cost_attribution_enabled(): + verbose_router_logger.warning( + "PTU fields are set on config.yaml deployment(s) %s, but PTU cost attribution is disabled, so no " + "flat cost accrues and this traffic is billed per token. Set %s=True to enable it", + ", ".join(ptu_declared), + PTU_COST_ATTRIBUTION_ENV_VAR, + ) + for model in original_model_list: _model_name = model.pop("model_name") _litellm_params = model.pop("litellm_params") diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 1580ec7f437..eb454bedbd8 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -8,6 +8,7 @@ should still use the built-in pricing. """ import copy +import logging import os import re import sys @@ -2007,3 +2008,127 @@ def test_a_falsy_id_is_still_scanned_for_collisions(): }, ] ) + + +# --- a reservation declared while the feature is off says so ------------------------ + + +def _ptu_warnings(caplog): + return tuple( + record.getMessage() + for record in caplog.records + if record.name == "LiteLLM Router" and record.levelno == logging.WARNING and "PTU" in record.getMessage() + ) + + +def test_a_reservation_declared_while_the_feature_is_off_is_warned_about(caplog): + """The deployment serves and bills per token, so without this the operator believes they + reserved capacity and sees no signal anywhere that nothing accrues.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(ptu_enabled=False) + + warnings = _ptu_warnings(caplog) + + assert len(warnings) == 1 + assert "gpt-4o-ptu" in warnings[0] + assert "LITELLM_ENABLE_PTU_COST_ATTRIBUTION" in warnings[0] + + +def test_a_reservation_is_not_warned_about_while_the_feature_is_on(caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router() + + assert _ptu_warnings(caplog) == () + + +def test_a_deployment_carrying_no_ptu_field_is_not_warned_about(caplog): + """Most of every config.yaml, so warning here would fire on proxies that never asked.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info={"team_id": "team-alpha"}, ptu_enabled=False) + + assert _ptu_warnings(caplog) == () + + +def test_a_half_written_reservation_is_warned_about(caplog): + """A count with no rate is not a chargeable reservation, but the operator still meant to + declare one, so what they wrote is what decides whether they hear about it.""" + half_written = {k: v for k, v in _PTU_MODEL_INFO.items() if k != "cost_per_ptu_per_hour"} + + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info=half_written, ptu_enabled=False) + + assert len(_ptu_warnings(caplog)) == 1 + + +@pytest.mark.parametrize( + "typo", + [ + {"ptu_count": 0}, + {"ptu_count": 0, "cost_per_ptu_per_hour": 0, "ptu_effective_from": None}, + ], + ids=["count out of range", "every value still a zero placeholder"], +) +def test_a_reservation_dropped_by_a_typo_is_warned_about(caplog, typo): + """An out-of-range value fails ModelInfo before the flag is ever consulted, so the + deployment stops serving on a proxy that never enabled PTU. The warning is what tells the + operator which feature the entry that vanished belonged to. + + Built the way proxy_server builds it, since dropping rather than raising is what + ``ignore_invalid_deployments`` does and config.yaml is loaded with it on. + """ + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": ""}, clear=False): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + router = Router( + ignore_invalid_deployments=True, + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://e.azure.com"}, + "model_info": {**_PTU_MODEL_INFO, **typo}, + } + ], + ) + + assert router.model_list == [] + assert len(_ptu_warnings(caplog)) == 1 + + +def test_a_db_backed_reservation_is_not_warned_about(caplog): + """/model/new already answered the caller with a 400, so repeating it on every reload + would report the operator's own rejected write back to them as a standing problem.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info={**_PTU_MODEL_INFO, "db_model": True}, ptu_enabled=False) + + assert _ptu_warnings(caplog) == () + + +def test_every_declaring_deployment_is_named(caplog): + """One line naming all of them, so a reload does not bury the config in repeats.""" + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": ""}, clear=False): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + Router( + model_list=[ + { + "model_name": "azure-ptu-east", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://e.azure.com"}, + "model_info": dict(_PTU_MODEL_INFO), + }, + { + "model_name": "azure-ptu-west", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://w.azure.com"}, + "model_info": {**_PTU_MODEL_INFO, "id": "ptu-alpha-westus"}, + }, + { + "model_name": "plain-gpt-4o", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://p.azure.com"}, + "model_info": {"id": "plain"}, + }, + ] + ) + + warnings = _ptu_warnings(caplog) + + assert len(warnings) == 1 + assert "azure-ptu-east" in warnings[0] + assert "azure-ptu-west" in warnings[0] + assert "plain-gpt-4o" not in warnings[0]