Merge 0f3194cb04 into 982d3a5476

litellm/router.py

@@ -524,6 +524,36 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(
 )
 
 
+_NO_PROMOTED_FIELDS: Final[Mapping[str, Any]] = MappingProxyType({})
+_NESTED_MODEL_INFO_WARNING: Final = (
+    "model=%s: 'model_info' is nested inside 'litellm_params'. It belongs at the deployment "
+    "level; applying it anyway, but please move it."
+)
+
+
+def _model_info_nested_under_litellm_params(
+    litellm_params: Mapping[str, Any], model_info: Mapping[str, Any]
+) -> Mapping[str, Any]:
+    """
+    Fields from a `model_info` block misplaced inside `litellm_params`, which would otherwise
+    be dropped: `model_info` is a declared field on GenericLiteLLMParams, so the misplaced
+    block validates and is then ignored by every cost path.
+
+    Deployment-level values win, and a nested `id` never displaces the generated
+    deployment id.
+    """
+    nested: Final = litellm_params.get("model_info")
+    if not isinstance(nested, dict):
+        return _NO_PROMOTED_FIELDS
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in nested.items()
+            if key != "id" and value is not None and model_info.get(key) is None
+        }
+    )
+
+
 class Router:
     model_names: set = set()
     cache_responses: bool | None = False
@@ -8139,6 +8169,10 @@ class Router:
         - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
         """
         try:
+            _promoted_model_info: Final = _model_info_nested_under_litellm_params(_litellm_params, _model_info)
+            if _promoted_model_info:
+                verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, _model_name)
+                _model_info.update(_promoted_model_info)
             config_sourced: Final = _model_info.get("db_model") is not True
             identity_error: Final = (
                 ptu_identity_error(
@@ -8904,6 +8938,14 @@ class Router:
             litellm_params=deployment.litellm_params.model_dump(exclude_none=True),
         )
 
+        _promoted_model_info: Final = _model_info_nested_under_litellm_params(
+            deployment.litellm_params.model_dump(), deployment.model_info.model_dump()
+        )
+        if _promoted_model_info:
+            verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, deployment.model_name)
+            for _key, _value in _promoted_model_info.items():
+                setattr(deployment.model_info, _key, _value)
+
         # add to model list
         _deployment: Final = deployment.to_json(exclude_none=True)
         # initialize client

tests/test_litellm/test_router_nested_model_info_pricing.py

@@ -0,0 +1,177 @@
+import pytest
+
+import litellm
+from litellm import Router
+from litellm.router import _model_info_nested_under_litellm_params
+
+BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731"
+PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731"
+INPUT_COST = 7.2e-08
+OUTPUT_COST = 1.44e-07
+CACHE_READ_COST = 1.44e-08
+
+
+def _pricing_model_info():
+    return {
+        "input_cost_per_token": INPUT_COST,
+        "output_cost_per_token": OUTPUT_COST,
+        "cache_read_input_token_cost": CACHE_READ_COST,
+    }
+
+
+def _nested_config():
+    """The reporter's config shape: model_info nested inside litellm_params."""
+    return {
+        "model_name": PUBLIC_MODEL,
+        "litellm_params": {
+            "model": BACKEND_MODEL,
+            "api_key": "fake-key",
+            "extra_body": {"service_tier": "flex"},
+            "model_info": _pricing_model_info(),
+        },
+    }
+
+
+def _deployment_level_config():
+    return {
+        "model_name": PUBLIC_MODEL,
+        "litellm_params": {
+            "model": BACKEND_MODEL,
+            "api_key": "fake-key",
+            "extra_body": {"service_tier": "flex"},
+        },
+        "model_info": _pricing_model_info(),
+    }
+
+
+@pytest.fixture(autouse=True)
+def clean_cost_map():
+    """Keep registrations from bleeding across tests."""
+    from litellm.utils import _invalidate_model_cost_lowercase_map
+
+    def _clear():
+        litellm.model_cost.pop(BACKEND_MODEL, None)
+        litellm.model_cost.pop(PUBLIC_MODEL, None)
+        _invalidate_model_cost_lowercase_map()
+
+    _clear()
+    yield
+    _clear()
+
+
+def _deployment_pricing(router):
+    deployment = router.model_list[0]
+    return deployment.get("model_info", {})
+
+
+def test_promotes_nothing_when_no_nested_block():
+    """No nested block, and a non-dict nested value, both promote nothing."""
+    assert _model_info_nested_under_litellm_params({"model": BACKEND_MODEL}, {}) == {}
+    assert _model_info_nested_under_litellm_params({"model_info": None}, {}) == {}
+    assert _model_info_nested_under_litellm_params({"model_info": "nope"}, {}) == {}
+
+
+def test_promotes_only_fields_the_deployment_does_not_define():
+    """Deployment-level values win; only unset fields are promoted."""
+    promoted = _model_info_nested_under_litellm_params(
+        {"model_info": {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}},
+        {"input_cost_per_token": INPUT_COST},
+    )
+
+    assert promoted == {"output_cost_per_token": 2.0}
+
+
+def test_never_promotes_id_and_skips_none_values():
+    """`id` is reserved for the generated deployment id, and None is not a value."""
+    promoted = _model_info_nested_under_litellm_params(
+        {
+            "model_info": {
+                "id": "nested-id",
+                "output_cost_per_token": None,
+                "input_cost_per_token": INPUT_COST,
+            }
+        },
+        {},
+    )
+
+    assert promoted == {"input_cost_per_token": INPUT_COST}
+
+
+def test_promoted_mapping_is_read_only():
+    """The caller updates its own dict; the returned view must not be mutable."""
+    promoted = _model_info_nested_under_litellm_params({"model_info": {"input_cost_per_token": INPUT_COST}}, {})
+
+    with pytest.raises(TypeError):
+        promoted["input_cost_per_token"] = 0.0  # type: ignore[index]
+
+
+def test_nested_model_info_pricing_is_applied():
+    """
+    Regression test for https://github.com/BerriAI/litellm/issues/35691
+
+    `model_info` is a declared field on GenericLiteLLMParams, so nesting it under
+    `litellm_params` validates cleanly and is then silently ignored by every cost
+    path, leaving the deployment with no pricing at all.
+    """
+    router = Router(model_list=[_nested_config()])
+    model_info = _deployment_pricing(router)
+
+    assert model_info.get("input_cost_per_token") == INPUT_COST
+    assert model_info.get("output_cost_per_token") == OUTPUT_COST
+    assert model_info.get("cache_read_input_token_cost") == CACHE_READ_COST
+
+
+def test_nested_matches_deployment_level():
+    """Both spellings must end up with the same pricing."""
+    nested = _deployment_pricing(Router(model_list=[_nested_config()]))
+    proper = _deployment_pricing(Router(model_list=[_deployment_level_config()]))
+
+    for field in (
+        "input_cost_per_token",
+        "output_cost_per_token",
+        "cache_read_input_token_cost",
+    ):
+        assert nested.get(field) == proper.get(field)
+
+
+def test_deployment_level_model_info_wins_over_nested():
+    """An explicit deployment-level value is not overwritten by a nested one."""
+    config = _deployment_level_config()
+    config["litellm_params"]["model_info"] = {"input_cost_per_token": 999.0}
+
+    model_info = _deployment_pricing(Router(model_list=[config]))
+    assert model_info.get("input_cost_per_token") == INPUT_COST
+
+
+def test_nested_model_info_does_not_override_deployment_id():
+    """The generated deployment id must survive a nested `id`."""
+    config = _nested_config()
+    config["litellm_params"]["model_info"]["id"] = "nested-id-should-be-ignored"
+
+    model_info = _deployment_pricing(Router(model_list=[config]))
+    assert model_info.get("id") != "nested-id-should-be-ignored"
+
+
+def test_no_nested_model_info_is_unchanged():
+    """Configs without the nested block keep working exactly as before."""
+    model_info = _deployment_pricing(Router(model_list=[_deployment_level_config()]))
+    assert model_info.get("input_cost_per_token") == INPUT_COST
+
+
+@pytest.mark.asyncio
+async def test_nested_model_info_produces_non_zero_response_cost():
+    """
+    End to end: the persisted spend is derived from `response_cost`, so pricing that
+    never reaches the cost map shows up as spend 0 in LiteLLM_SpendLogs.
+    """
+    router = Router(model_list=[_nested_config()])
+
+    response = await router.acompletion(
+        model=PUBLIC_MODEL,
+        messages=[{"role": "user", "content": "hi"}],
+        mock_response="hello",
+    )
+
+    response_cost = response._hidden_params.get("response_cost")
+    assert response_cost is not None
+    assert response_cost > 0