From 45efd07c691ad70d0cacd6b90be436fa1106cdd9 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Thu, 6 Aug 2026 01:11:35 +0530 Subject: [PATCH 1/2] fix(router): apply model_info nested under litellm_params instead of dropping it model_info is a declared field on GenericLiteLLMParams, so a config.yaml that nests it inside litellm_params validates cleanly and is then silently ignored by every cost path. The deployment ends up with no pricing at all, so response_cost is 0 and the LiteLLM_SpendLogs row records spend 0 even though the requests cost real money Router now promotes a nested model_info onto the deployment, warning that it belongs at the deployment level. Explicit deployment-level values win, and a nested id cannot displace the generated deployment id. add_deployment gets the same treatment since DB-loaded and /model/new deployments do not go through _create_deployment Fixes #35691 --- litellm/router.py | 42 +++++ .../test_router_nested_model_info_pricing.py | 177 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 tests/unit/test_router_nested_model_info_pricing.py diff --git a/litellm/router.py b/litellm/router.py index 6f416c416c0..bad3a5d8821 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -751,6 +751,36 @@ def as_output_cap(value: object) -> int | None: return cap if cap >= 0 else None +_NO_PROMOTED_FIELDS: Final[Mapping[str, Any]] = MappingProxyType({}) +_NESTED_MODEL_INFO_WARNING: Final = ( + "model=%s: 'model_info' is nested inside 'litellm_params'. It belongs at the deployment " + "level; applying it anyway, but please move it." +) + + +def _model_info_nested_under_litellm_params( + litellm_params: Mapping[str, Any], model_info: Mapping[str, Any] +) -> Mapping[str, Any]: + """ + Fields from a `model_info` block misplaced inside `litellm_params`, which would otherwise + be dropped: `model_info` is a declared field on GenericLiteLLMParams, so the misplaced + block validates and is then ignored by every cost path. + + Deployment-level values win, and a nested `id` never displaces the generated + deployment id. + """ + nested: Final = litellm_params.get("model_info") + if not isinstance(nested, dict): + return _NO_PROMOTED_FIELDS + return MappingProxyType( + { + key: value + for key, value in nested.items() + if key != "id" and value is not None and model_info.get(key) is None + } + ) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -8822,6 +8852,10 @@ class Router: - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params) """ try: + _promoted_model_info: Final = _model_info_nested_under_litellm_params(_litellm_params, _model_info) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, _model_name) + _model_info.update(_promoted_model_info) config_sourced: Final = _model_info.get("db_model") is not True identity_error: Final = ( ptu_identity_error( @@ -9650,6 +9684,14 @@ class Router: litellm_params=deployment.litellm_params.model_dump(exclude_none=True), ) + _promoted_model_info: Final = _model_info_nested_under_litellm_params( + deployment.litellm_params.model_dump(), deployment.model_info.model_dump() + ) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, deployment.model_name) + for _key, _value in _promoted_model_info.items(): + setattr(deployment.model_info, _key, _value) + # add to model list _deployment: Final = deployment.to_json(exclude_none=True) # initialize client diff --git a/tests/unit/test_router_nested_model_info_pricing.py b/tests/unit/test_router_nested_model_info_pricing.py new file mode 100644 index 00000000000..02412b49b0c --- /dev/null +++ b/tests/unit/test_router_nested_model_info_pricing.py @@ -0,0 +1,177 @@ +import pytest + +import litellm +from litellm import Router +from litellm.router import _model_info_nested_under_litellm_params + +BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731" +PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" +INPUT_COST = 7.2e-08 +OUTPUT_COST = 1.44e-07 +CACHE_READ_COST = 1.44e-08 + + +def _pricing_model_info(): + return { + "input_cost_per_token": INPUT_COST, + "output_cost_per_token": OUTPUT_COST, + "cache_read_input_token_cost": CACHE_READ_COST, + } + + +def _nested_config(): + """The reporter's config shape: model_info nested inside litellm_params.""" + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + "model_info": _pricing_model_info(), + }, + } + + +def _deployment_level_config(): + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + }, + "model_info": _pricing_model_info(), + } + + +@pytest.fixture(autouse=True) +def clean_cost_map(): + """Keep registrations from bleeding across tests.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + + def _clear(): + litellm.model_cost.pop(BACKEND_MODEL, None) + litellm.model_cost.pop(PUBLIC_MODEL, None) + _invalidate_model_cost_lowercase_map() + + _clear() + yield + _clear() + + +def _deployment_pricing(router): + deployment = router.model_list[0] + return deployment.get("model_info", {}) + + +def test_promotes_nothing_when_no_nested_block(): + """No nested block, and a non-dict nested value, both promote nothing.""" + assert _model_info_nested_under_litellm_params({"model": BACKEND_MODEL}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": None}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": "nope"}, {}) == {} + + +def test_promotes_only_fields_the_deployment_does_not_define(): + """Deployment-level values win; only unset fields are promoted.""" + promoted = _model_info_nested_under_litellm_params( + {"model_info": {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}}, + {"input_cost_per_token": INPUT_COST}, + ) + + assert promoted == {"output_cost_per_token": 2.0} + + +def test_never_promotes_id_and_skips_none_values(): + """`id` is reserved for the generated deployment id, and None is not a value.""" + promoted = _model_info_nested_under_litellm_params( + { + "model_info": { + "id": "nested-id", + "output_cost_per_token": None, + "input_cost_per_token": INPUT_COST, + } + }, + {}, + ) + + assert promoted == {"input_cost_per_token": INPUT_COST} + + +def test_promoted_mapping_is_read_only(): + """The caller updates its own dict; the returned view must not be mutable.""" + promoted = _model_info_nested_under_litellm_params({"model_info": {"input_cost_per_token": INPUT_COST}}, {}) + + with pytest.raises(TypeError): + promoted["input_cost_per_token"] = 0.0 # type: ignore[index] + + +def test_nested_model_info_pricing_is_applied(): + """ + Regression test for https://github.com/BerriAI/litellm/issues/35691 + + `model_info` is a declared field on GenericLiteLLMParams, so nesting it under + `litellm_params` validates cleanly and is then silently ignored by every cost + path, leaving the deployment with no pricing at all. + """ + router = Router(model_list=[_nested_config()]) + model_info = _deployment_pricing(router) + + assert model_info.get("input_cost_per_token") == INPUT_COST + assert model_info.get("output_cost_per_token") == OUTPUT_COST + assert model_info.get("cache_read_input_token_cost") == CACHE_READ_COST + + +def test_nested_matches_deployment_level(): + """Both spellings must end up with the same pricing.""" + nested = _deployment_pricing(Router(model_list=[_nested_config()])) + proper = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + ): + assert nested.get(field) == proper.get(field) + + +def test_deployment_level_model_info_wins_over_nested(): + """An explicit deployment-level value is not overwritten by a nested one.""" + config = _deployment_level_config() + config["litellm_params"]["model_info"] = {"input_cost_per_token": 999.0} + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +def test_nested_model_info_does_not_override_deployment_id(): + """The generated deployment id must survive a nested `id`.""" + config = _nested_config() + config["litellm_params"]["model_info"]["id"] = "nested-id-should-be-ignored" + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("id") != "nested-id-should-be-ignored" + + +def test_no_nested_model_info_is_unchanged(): + """Configs without the nested block keep working exactly as before.""" + model_info = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +@pytest.mark.asyncio +async def test_nested_model_info_produces_non_zero_response_cost(): + """ + End to end: the persisted spend is derived from `response_cost`, so pricing that + never reaches the cost map shows up as spend 0 in LiteLLM_SpendLogs. + """ + router = Router(model_list=[_nested_config()]) + + response = await router.acompletion( + model=PUBLIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + ) + + response_cost = response._hidden_params.get("response_cost") + assert response_cost is not None + assert response_cost > 0 From 25be3c19a0f6e8f78ace16308a3a988245ea4b73 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:18:21 +0530 Subject: [PATCH 2/2] test(router): cover model_info promotion in add_deployment DB-loaded and /model/new deployments reach the router through add_deployment, so pin the promotion there too The cost-map fixture now puts back the entries it removes. The shipped map prices the backend model, and popping that entry for good left it unpriced for every later test in the same process Co-Authored-By: Claude Opus 5.5 --- .../test_router_nested_model_info_pricing.py | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_router_nested_model_info_pricing.py b/tests/unit/test_router_nested_model_info_pricing.py index 02412b49b0c..4657a47e5ca 100644 --- a/tests/unit/test_router_nested_model_info_pricing.py +++ b/tests/unit/test_router_nested_model_info_pricing.py @@ -3,12 +3,14 @@ import pytest import litellm from litellm import Router from litellm.router import _model_info_nested_under_litellm_params +from litellm.types.router import Deployment, LiteLLM_Params BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731" PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" INPUT_COST = 7.2e-08 OUTPUT_COST = 1.44e-07 CACHE_READ_COST = 1.44e-08 +DEPLOYMENT_ID = "db-deployment-id" def _pricing_model_info(): @@ -46,17 +48,23 @@ def _deployment_level_config(): @pytest.fixture(autouse=True) def clean_cost_map(): - """Keep registrations from bleeding across tests.""" + """Keep registrations from bleeding across tests, and put back the entries the cost map already had.""" from litellm.utils import _invalidate_model_cost_lowercase_map + cost_map = litellm.model_cost + keys = (BACKEND_MODEL, PUBLIC_MODEL, DEPLOYMENT_ID) + original = {key: cost_map[key] for key in keys if key in cost_map} + def _clear(): - litellm.model_cost.pop(BACKEND_MODEL, None) - litellm.model_cost.pop(PUBLIC_MODEL, None) + for key in keys: + cost_map.pop(key, None) _invalidate_model_cost_lowercase_map() _clear() yield _clear() + cost_map.update(original) + _invalidate_model_cost_lowercase_map() def _deployment_pricing(router): @@ -64,6 +72,16 @@ def _deployment_pricing(router): return deployment.get("model_info", {}) +def _nested_deployment(model_info): + """A deployment built the way the proxy loads one from the DB, nested block included.""" + config = _nested_config() + return Deployment( + model_name=config["model_name"], + litellm_params=LiteLLM_Params.model_validate(config["litellm_params"]), + model_info=model_info, + ) + + def test_promotes_nothing_when_no_nested_block(): """No nested block, and a non-dict nested value, both promote nothing.""" assert _model_info_nested_under_litellm_params({"model": BACKEND_MODEL}, {}) == {} @@ -175,3 +193,45 @@ async def test_nested_model_info_produces_non_zero_response_cost(): response_cost = response._hidden_params.get("response_cost") assert response_cost is not None assert response_cost > 0 + + +@pytest.mark.asyncio +async def test_add_deployment_applies_nested_model_info_pricing(): + """ + DB-loaded and /model/new deployments reach the router through add_deployment, + not _create_deployment, so the nested block has to be applied there as well. + """ + router = Router(model_list=[]) + assert router.add_deployment(_nested_deployment({"id": DEPLOYMENT_ID})) is not None + + model_info = _deployment_pricing(router) + assert model_info.get("input_cost_per_token") == INPUT_COST + assert model_info.get("output_cost_per_token") == OUTPUT_COST + assert model_info.get("cache_read_input_token_cost") == CACHE_READ_COST + + # Exact, not just > 0: the shipped cost map also prices this backend model. + response = await router.acompletion( + model=PUBLIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + ) + usage = response.usage + assert response._hidden_params.get("response_cost") == pytest.approx( + usage.prompt_tokens * INPUT_COST + usage.completion_tokens * OUTPUT_COST + ) + + +def test_add_deployment_keeps_deployment_level_values_and_id(): + """Only unset fields are applied, and the nested `id` never re-keys the deployment.""" + deployment = _nested_deployment({"id": DEPLOYMENT_ID, "input_cost_per_token": INPUT_COST}) + deployment.litellm_params.model_info.update(id="nested-id-should-be-ignored", input_cost_per_token=999.0) + + router = Router(model_list=[]) + router.add_deployment(deployment) + + model_info = _deployment_pricing(router) + assert model_info.get("input_cost_per_token") == INPUT_COST + assert model_info.get("output_cost_per_token") == OUTPUT_COST + assert model_info.get("id") == DEPLOYMENT_ID + assert router.has_model_id(DEPLOYMENT_ID) + assert not router.has_model_id("nested-id-should-be-ignored")