From d7b4ea02731d6e1c2b8705e19346d7bf62b6b94f Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Thu, 6 Aug 2026 01:11:35 +0530 Subject: [PATCH 1/3] fix(router): apply model_info nested under litellm_params instead of dropping it model_info is a declared field on GenericLiteLLMParams, so a config.yaml that nests it inside litellm_params validates cleanly and is then silently ignored by every cost path. The deployment ends up with no pricing at all, so response_cost is 0 and the LiteLLM_SpendLogs row records spend 0 even though the requests cost real money Router now promotes a nested model_info onto the deployment, warning that it belongs at the deployment level. Explicit deployment-level values win, and a nested id cannot displace the generated deployment id. add_deployment gets the same treatment since DB-loaded and /model/new deployments do not go through _create_deployment Fixes #35691 --- litellm/router.py | 42 ++++++ .../test_router_nested_model_info_pricing.py | 135 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 tests/test_litellm/test_router_nested_model_info_pricing.py diff --git a/litellm/router.py b/litellm/router.py index 26158ae0a56..8d1972ca641 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -371,6 +371,36 @@ set_live_deployment_replay(_replay_live_router_model_cost) RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(("messages", "original_function", "attempted_targets")) +_NO_PROMOTED_FIELDS: Final[Mapping[str, Any]] = MappingProxyType({}) +_NESTED_MODEL_INFO_WARNING: Final = ( + "model=%s: 'model_info' is nested inside 'litellm_params'. It belongs at the deployment " + "level; applying it anyway, but please move it." +) + + +def _model_info_nested_under_litellm_params( + litellm_params: Mapping[str, Any], model_info: Mapping[str, Any] +) -> Mapping[str, Any]: + """ + Fields from a `model_info` block misplaced inside `litellm_params`, which would otherwise + be dropped: `model_info` is a declared field on GenericLiteLLMParams, so the misplaced + block validates and is then ignored by every cost path. + + See https://github.com/BerriAI/litellm/issues/35691. Deployment-level values win, and a + nested `id` never displaces the generated deployment id. + """ + nested: Final = litellm_params.get("model_info") + if not isinstance(nested, dict): + return _NO_PROMOTED_FIELDS + return MappingProxyType( + { + key: value + for key, value in nested.items() + if key != "id" and value is not None and model_info.get(key) is None + } + ) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -7696,6 +7726,10 @@ class Router: - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params) """ try: + _promoted_model_info: Final = _model_info_nested_under_litellm_params(_litellm_params, _model_info) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, _model_name) + _model_info.update(_promoted_model_info) # rebind-ok: this dict is populated in place below too zeroed_pricing: Final = ( zeroed_ptu_pricing(_model_info, _litellm_params) if _model_info.get("db_model") is not True else None ) @@ -8418,6 +8452,14 @@ class Router: litellm_params=deployment.litellm_params.model_dump(exclude_none=True), ) + _promoted_model_info: Final = _model_info_nested_under_litellm_params( + deployment.litellm_params.model_dump(), deployment.model_info.model_dump() + ) + if _promoted_model_info: + verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, deployment.model_name) + for _key, _value in _promoted_model_info.items(): + setattr(deployment.model_info, _key, _value) + # add to model list _deployment: Final = deployment.to_json(exclude_none=True) # initialize client diff --git a/tests/test_litellm/test_router_nested_model_info_pricing.py b/tests/test_litellm/test_router_nested_model_info_pricing.py new file mode 100644 index 00000000000..c266b4c73f9 --- /dev/null +++ b/tests/test_litellm/test_router_nested_model_info_pricing.py @@ -0,0 +1,135 @@ +import pytest + +import litellm +from litellm import Router + +BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731" +PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" +INPUT_COST = 7.2e-08 +OUTPUT_COST = 1.44e-07 +CACHE_READ_COST = 1.44e-08 + + +def _pricing_model_info(): + return { + "input_cost_per_token": INPUT_COST, + "output_cost_per_token": OUTPUT_COST, + "cache_read_input_token_cost": CACHE_READ_COST, + } + + +def _nested_config(): + """The reporter's config shape: model_info nested inside litellm_params.""" + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + "model_info": _pricing_model_info(), + }, + } + + +def _deployment_level_config(): + return { + "model_name": PUBLIC_MODEL, + "litellm_params": { + "model": BACKEND_MODEL, + "api_key": "fake-key", + "extra_body": {"service_tier": "flex"}, + }, + "model_info": _pricing_model_info(), + } + + +@pytest.fixture(autouse=True) +def clean_cost_map(): + """Keep registrations from bleeding across tests.""" + from litellm.utils import _invalidate_model_cost_lowercase_map + + def _clear(): + litellm.model_cost.pop(BACKEND_MODEL, None) + litellm.model_cost.pop(PUBLIC_MODEL, None) + _invalidate_model_cost_lowercase_map() + + _clear() + yield + _clear() + + +def _deployment_pricing(router): + deployment = router.model_list[0] + return deployment.get("model_info", {}) + + +def test_nested_model_info_pricing_is_applied(): + """ + Regression test for https://github.com/BerriAI/litellm/issues/35691 + + `model_info` is a declared field on GenericLiteLLMParams, so nesting it under + `litellm_params` validates cleanly and is then silently ignored by every cost + path, leaving the deployment with no pricing at all. + """ + router = Router(model_list=[_nested_config()]) + model_info = _deployment_pricing(router) + + assert model_info.get("input_cost_per_token") == INPUT_COST + assert model_info.get("output_cost_per_token") == OUTPUT_COST + assert model_info.get("cache_read_input_token_cost") == CACHE_READ_COST + + +def test_nested_matches_deployment_level(): + """Both spellings must end up with the same pricing.""" + nested = _deployment_pricing(Router(model_list=[_nested_config()])) + proper = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + ): + assert nested.get(field) == proper.get(field) + + +def test_deployment_level_model_info_wins_over_nested(): + """An explicit deployment-level value is not overwritten by a nested one.""" + config = _deployment_level_config() + config["litellm_params"]["model_info"] = {"input_cost_per_token": 999.0} + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +def test_nested_model_info_does_not_override_deployment_id(): + """The generated deployment id must survive a nested `id`.""" + config = _nested_config() + config["litellm_params"]["model_info"]["id"] = "nested-id-should-be-ignored" + + model_info = _deployment_pricing(Router(model_list=[config])) + assert model_info.get("id") != "nested-id-should-be-ignored" + + +def test_no_nested_model_info_is_unchanged(): + """Configs without the nested block keep working exactly as before.""" + model_info = _deployment_pricing(Router(model_list=[_deployment_level_config()])) + assert model_info.get("input_cost_per_token") == INPUT_COST + + +@pytest.mark.asyncio +async def test_nested_model_info_produces_non_zero_response_cost(): + """ + End to end: the persisted spend is derived from `response_cost`, so pricing that + never reaches the cost map shows up as spend 0 in LiteLLM_SpendLogs. + """ + router = Router(model_list=[_nested_config()]) + + response = await router.acompletion( + model=PUBLIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + ) + + response_cost = response._hidden_params.get("response_cost") + assert response_cost is not None + assert response_cost > 0 From d559f440c5626461e625a5a7e28797d68860a90e Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:35:58 +0530 Subject: [PATCH 2/3] chore(router): drop comments the repo guidance prohibits Review follow-up: CLAUDE.md says no new code comments unless asked, and the discipline checker does not require a rebind-ok marker on this line (verified: its output is identical with and without). The issue link lives in the PR body already --- litellm/router.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8d1972ca641..8662ab77cf9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -386,8 +386,8 @@ def _model_info_nested_under_litellm_params( be dropped: `model_info` is a declared field on GenericLiteLLMParams, so the misplaced block validates and is then ignored by every cost path. - See https://github.com/BerriAI/litellm/issues/35691. Deployment-level values win, and a - nested `id` never displaces the generated deployment id. + Deployment-level values win, and a nested `id` never displaces the generated + deployment id. """ nested: Final = litellm_params.get("model_info") if not isinstance(nested, dict): @@ -7729,7 +7729,7 @@ class Router: _promoted_model_info: Final = _model_info_nested_under_litellm_params(_litellm_params, _model_info) if _promoted_model_info: verbose_router_logger.warning(_NESTED_MODEL_INFO_WARNING, _model_name) - _model_info.update(_promoted_model_info) # rebind-ok: this dict is populated in place below too + _model_info.update(_promoted_model_info) zeroed_pricing: Final = ( zeroed_ptu_pricing(_model_info, _litellm_params) if _model_info.get("db_model") is not True else None ) From 9b61730fe4bed5aac1884444e8c9aed3b3e945a7 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:58:36 +0530 Subject: [PATCH 3/3] test(router): cover _model_info_nested_under_litellm_params directly router_code_coverage.py requires every function in router.py to be called by name from a router-named test file. The existing tests exercised the helper only through Router, so the new function counted as untested and the code-quality check failed The four added cases pin the helper's contract rather than merely naming it: absent or non-dict nested block promotes nothing, deployment-level values win over nested ones, id is never promoted and None is not a value, and the returned mapping is read-only --- .../test_router_nested_model_info_pricing.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_litellm/test_router_nested_model_info_pricing.py b/tests/test_litellm/test_router_nested_model_info_pricing.py index c266b4c73f9..02412b49b0c 100644 --- a/tests/test_litellm/test_router_nested_model_info_pricing.py +++ b/tests/test_litellm/test_router_nested_model_info_pricing.py @@ -2,6 +2,7 @@ import pytest import litellm from litellm import Router +from litellm.router import _model_info_nested_under_litellm_params BACKEND_MODEL = "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731" PUBLIC_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" @@ -63,6 +64,47 @@ def _deployment_pricing(router): return deployment.get("model_info", {}) +def test_promotes_nothing_when_no_nested_block(): + """No nested block, and a non-dict nested value, both promote nothing.""" + assert _model_info_nested_under_litellm_params({"model": BACKEND_MODEL}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": None}, {}) == {} + assert _model_info_nested_under_litellm_params({"model_info": "nope"}, {}) == {} + + +def test_promotes_only_fields_the_deployment_does_not_define(): + """Deployment-level values win; only unset fields are promoted.""" + promoted = _model_info_nested_under_litellm_params( + {"model_info": {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}}, + {"input_cost_per_token": INPUT_COST}, + ) + + assert promoted == {"output_cost_per_token": 2.0} + + +def test_never_promotes_id_and_skips_none_values(): + """`id` is reserved for the generated deployment id, and None is not a value.""" + promoted = _model_info_nested_under_litellm_params( + { + "model_info": { + "id": "nested-id", + "output_cost_per_token": None, + "input_cost_per_token": INPUT_COST, + } + }, + {}, + ) + + assert promoted == {"input_cost_per_token": INPUT_COST} + + +def test_promoted_mapping_is_read_only(): + """The caller updates its own dict; the returned view must not be mutable.""" + promoted = _model_info_nested_under_litellm_params({"model_info": {"input_cost_per_token": INPUT_COST}}, {}) + + with pytest.raises(TypeError): + promoted["input_cost_per_token"] = 0.0 # type: ignore[index] + + def test_nested_model_info_pricing_is_applied(): """ Regression test for https://github.com/BerriAI/litellm/issues/35691