diff --git a/litellm/router.py b/litellm/router.py index f0ebb539bb7..8083442e396 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -524,6 +524,36 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( ) +_NO_PROMOTED_FIELDS: Final[Mapping[str, Any]] = MappingProxyType({}) +_NESTED_MODEL_INFO_WARNING: Final = ( + "model=%s: 'model_info' is nested inside 'litellm_params'. It belongs at the deployment " + "level; applying it anyway, but please move it." +) + + +def _model_info_nested_under_litellm_params( + litellm_params: Mapping[str, Any], model_info: Mapping[str, Any] +) -> Mapping[str, Any]: + """ + Fields from a `model_info` block misplaced inside `litellm_params`, which would otherwise + be dropped: `model_info` is a declared field on GenericLiteLLMParams, so the misplaced + block validates and is then ignored by every cost path. + + Deployment-level values win, and a nested `id` never displaces the generated + deployment id. + """ + nested: Final = litellm_params.get("model_info") + if not isinstance(nested, dict): + return _NO_PROMOTED_FIELDS + return MappingProxyType( + { + key: value + for key, value in nested.items() + if key != "id" and value is not None and model_info.get(key) is None + } + ) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -8139,6 +8169,10 @@ class Router: - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params) """ try: + _promoted_model_info: Final = _model_info_nested_under_litellm_params(_litellm_params, _model_info) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, _model_name) + _model_info.update(_promoted_model_info) config_sourced: Final = _model_info.get("db_model") is not True identity_error: Final = ( ptu_identity_error( @@ -8904,6 +8938,14 @@ class Router: litellm_params=deployment.litellm_params.model_dump(exclude_none=True), ) + _promoted_model_info: Final = _model_info_nested_under_litellm_params( + deployment.litellm_params.model_dump(), deployment.model_info.model_dump() + ) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, deployment.model_name) + for _key, _value in _promoted_model_info.items(): + setattr(deployment.model_info, _key, _value) + # add to model list _deployment: Final = deployment.to_json(exclude_none=True) # initialize client diff --git a/tests/test_litellm/test_router_nested_model_info_pricing.py b/tests/test_litellm/test_router_nested_model_info_pricing.py new file mode 100644 index 00000000000..02412b49b0c --- /dev/null +++ b/tests/test_litellm/test_router_nested_model_info_pricing.py @@ -0,0 +1,177 @@ +import pytest + +import litellm +from litellm import Router +from litellm.router import _model_info_nested_under_litellm_params + +BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731" +PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" +INPUT_COST = 7.2e-08 +OUTPUT_COST = 1.44e-07 +CACHE_READ_COST = 1.44e-08 + + +def _pricing_model_info(): + return { + "input_cost_per_token": INPUT_COST, + "output_cost_per_token": OUTPUT_COST, + "cache_read_input_token_cost": CACHE_READ_COST, + } + + +def _nested_config(): + """The reporter's config shape: model_info nested inside litellm_params.""" + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + "model_info": _pricing_model_info(), + }, + } + + +def _deployment_level_config(): + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + }, + "model_info": _pricing_model_info(), + } + + +@pytest.fixture(autouse=True) +def clean_cost_map(): + """Keep registrations from bleeding across tests.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + + def _clear(): + litellm.model_cost.pop(BACKEND_MODEL, None) + litellm.model_cost.pop(PUBLIC_MODEL, None) + _invalidate_model_cost_lowercase_map() + + _clear() + yield + _clear() + + +def _deployment_pricing(router): + deployment = router.model_list[0] + return deployment.get("model_info", {}) + + +def test_promotes_nothing_when_no_nested_block(): + """No nested block, and a non-dict nested value, both promote nothing.""" + assert _model_info_nested_under_litellm_params({"model": BACKEND_MODEL}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": None}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": "nope"}, {}) == {} + + +def test_promotes_only_fields_the_deployment_does_not_define(): + """Deployment-level values win; only unset fields are promoted.""" + promoted = _model_info_nested_under_litellm_params( + {"model_info": {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}}, + {"input_cost_per_token": INPUT_COST}, + ) + + assert promoted == {"output_cost_per_token": 2.0} + + +def test_never_promotes_id_and_skips_none_values(): + """`id` is reserved for the generated deployment id, and None is not a value.""" + promoted = _model_info_nested_under_litellm_params( + { + "model_info": { + "id": "nested-id", + "output_cost_per_token": None, + "input_cost_per_token": INPUT_COST, + } + }, + {}, + ) + + assert promoted == {"input_cost_per_token": INPUT_COST} + + +def test_promoted_mapping_is_read_only(): + """The caller updates its own dict; the returned view must not be mutable.""" + promoted = _model_info_nested_under_litellm_params({"model_info": {"input_cost_per_token": INPUT_COST}}, {}) + + with pytest.raises(TypeError): + promoted["input_cost_per_token"] = 0.0 # type: ignore[index] + + +def test_nested_model_info_pricing_is_applied(): + """ + Regression test for https://github.com/BerriAI/litellm/issues/35691 + + `model_info` is a declared field on GenericLiteLLMParams, so nesting it under + `litellm_params` validates cleanly and is then silently ignored by every cost + path, leaving the deployment with no pricing at all. + """ + router = Router(model_list=[_nested_config()]) + model_info = _deployment_pricing(router) + + assert model_info.get("input_cost_per_token") == INPUT_COST + assert model_info.get("output_cost_per_token") == OUTPUT_COST + assert model_info.get("cache_read_input_token_cost") == CACHE_READ_COST + + +def test_nested_matches_deployment_level(): + """Both spellings must end up with the same pricing.""" + nested = _deployment_pricing(Router(model_list=[_nested_config()])) + proper = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + ): + assert nested.get(field) == proper.get(field) + + +def test_deployment_level_model_info_wins_over_nested(): + """An explicit deployment-level value is not overwritten by a nested one.""" + config = _deployment_level_config() + config["litellm_params"]["model_info"] = {"input_cost_per_token": 999.0} + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +def test_nested_model_info_does_not_override_deployment_id(): + """The generated deployment id must survive a nested `id`.""" + config = _nested_config() + config["litellm_params"]["model_info"]["id"] = "nested-id-should-be-ignored" + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("id") != "nested-id-should-be-ignored" + + +def test_no_nested_model_info_is_unchanged(): + """Configs without the nested block keep working exactly as before.""" + model_info = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +@pytest.mark.asyncio +async def test_nested_model_info_produces_non_zero_response_cost(): + """ + End to end: the persisted spend is derived from `response_cost`, so pricing that + never reaches the cost map shows up as spend 0 in LiteLLM_SpendLogs. + """ + router = Router(model_list=[_nested_config()]) + + response = await router.acompletion( + model=PUBLIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + ) + + response_cost = response._hidden_params.get("response_cost") + assert response_cost is not None + assert response_cost > 0