From a8f14c013dffc28038481059a39f6b410b009060 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:59:19 +0530 Subject: [PATCH 1/3] fix(utils): stop register_model materializing empty model_cost entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit register_model() ran litellm.model_cost.setdefault(key, {}) even when the merged payload was empty, so registering an unmapped model with an empty payload created a bare key. Key existence is what cost lookup treats as 'mapped', so completion_cost() for that model flipped from raising 'This model isn't mapped yet' to silently returning $0.0 — process-wide. Router.__init__ hits this for every deployment: the shared-backend registration (_register_deployment_in_model_cost) strips per-deployment pricing/metadata from the shared payload, so any deployment configured without pricing registers {} under its backend model name. Fix: only write to litellm.model_cost when the merged payload is non-empty. Test updates, both verified against pre-fix behavior: - test_register_model_router_add_deployment_custom_pricing_applies asserted on the backend keys, but pre-fix those held exactly the empty entry ({}), making _check_provider_match vacuously True; the custom pricing lives (and always lived) under the deployment id entry. The test now asserts on that entry. - test_repointing_a_deployment_drops_its_previous_backend_key / test_tiered_pricing_only_deployment_selects_router_model_id relied on the empty shared entry existing; the deployments now carry (or tolerate the absence of) a real shared field so the guarantees stay exercised. Co-Authored-By: Claude Fable 5 --- litellm/utils.py | 15 ++- tests/test_litellm/test_cost_calculator.py | 10 +- .../test_register_model_custom_pricing.py | 97 ++++++++++++++++--- .../test_router_model_cost_isolation.py | 10 +- 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 911de83b785..9702e239f56 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2807,12 +2807,19 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru existing_model.pop(_cost_field, None) ## override / add new keys to the existing model cost dictionary updated_dictionary = _update_dictionary(existing_model, value) - litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary) + # An empty payload for a model with no built-in entry carries no + # information — do not materialize a bare key for it. Key existence is + # what cost lookup treats as "mapped", so an empty entry flips + # ``completion_cost`` for an unmapped model from a loud "model isn't + # mapped" error to a silent $0.0 (``Router.__init__`` registers every + # deployment's backend key this way when no pricing is configured). + if updated_dictionary: + litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary) - # Invalidate case-insensitive lookup map since model_cost was modified - _invalidate_model_cost_lowercase_map() + # Invalidate case-insensitive lookup map since model_cost was modified + _invalidate_model_cost_lowercase_map() - verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) + verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key) # add new model names to provider lists if value.get("litellm_provider") == "openai": if key not in litellm.open_ai_chat_completion_models: diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 3f024e2fd03..ab134d97ef0 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,5 +1,6 @@ import os import sys +from typing import Final import pytest @@ -1036,10 +1037,11 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): entry = litellm.model_cost[router_model_id] assert entry.get("input_cost_per_token") is None assert entry.get("tiered_pricing") is not None - # The stripped shared alias must not carry tiered pricing. - assert ( - litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None - ) + # The stripped shared alias must not carry tiered pricing. After + # stripping, this deployment contributes nothing to the shared key, so + # ``register_model`` may not have materialized it at all. + shared_alias_entry: Final = litellm.model_cost.get("dashscope/qwen-tier-only-test") + assert shared_alias_entry is None or shared_alias_entry.get("tiered_pricing") is None selected = _select_model_name_for_cost_calc( model="dashscope/qwen-tier-only-test", diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py index ba82bfaadc6..ceb42130856 100644 --- a/tests/test_litellm/test_register_model_custom_pricing.py +++ b/tests/test_litellm/test_register_model_custom_pricing.py @@ -12,6 +12,7 @@ calculations for DB-sourced models with prompt caching pricing. import copy import os import sys +from typing import Final import pytest @@ -469,25 +470,28 @@ def test_register_model_router_add_deployment_custom_pricing_applies(): ) try: - # ``add_deployment`` runs as part of ``Router.__init__``; the - # registered entry must not block ``_check_provider_match`` for - # the deployment's provider. + # ``add_deployment`` runs as part of ``Router.__init__``; the custom + # pricing is registered under the deployment's ``model_info.id`` (the + # shared backend key deliberately excludes per-deployment pricing). + # That entry must not block ``_check_provider_match`` for the + # deployment's provider. from litellm.utils import _check_provider_match - registered_keys = [ - k for k in (deployment_model, model_key) if k in litellm.model_cost - ] - assert registered_keys, ( - "Router.add_deployment did not register custom pricing for " + registered: Final = litellm.model_cost.get("deployment-28336") + assert registered is not None, ( + "Router.add_deployment did not register custom pricing under " + "the deployment id for " f"{model_key} / {deployment_model}" ) - for k in registered_keys: - assert ( - _check_provider_match(litellm.model_cost[k], "openai") is True - ), f"custom pricing for {k} was dropped by _check_provider_match" + assert registered["input_cost_per_token"] == 0.00042 + assert registered["output_cost_per_token"] == 0.00084 + assert ( + _check_provider_match(registered, "openai") is True + ), "custom pricing for deployment-28336 was dropped by _check_provider_match" finally: litellm.model_cost.pop(model_key, None) litellm.model_cost.pop(deployment_model, None) + litellm.model_cost.pop("deployment-28336", None) del router @@ -652,3 +656,72 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): finally: litellm.model_cost.pop(model_key, None) _invalidate_model_cost_lowercase_map() + + +def test_register_model_empty_payload_does_not_materialize_entry(): + """An empty payload for a model with no built-in entry carries no + information, and ``register_model`` must not create a bare key for it in + ``litellm.model_cost``. Key existence is what cost lookup treats as + "mapped", so an empty entry flips ``completion_cost`` for an unmapped + model from a loud "model isn't mapped" error to a silent $0.0. + """ + from litellm.utils import _runtime_registered_model_cost + + model_key: Final = "deepinfra/fake-unmapped-model-empty-registration" + assert model_key not in litellm.model_cost + with pytest.raises(Exception, match="isn't mapped"): + litellm.completion_cost(model=model_key, prompt="hi", completion="there") + + try: + litellm.register_model({model_key: {}}) # mutable-ok: registration payload under test + + assert model_key not in litellm.model_cost, ( + "register_model materialized an empty model_cost entry for an " + "unmapped model registered with an empty payload" + ) + with pytest.raises(Exception, match="isn't mapped"): + litellm.completion_cost(model=model_key, prompt="hi", completion="there") + finally: + litellm.model_cost.pop(model_key, None) + _runtime_registered_model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_router_init_without_pricing_keeps_unmapped_model_cost_loud(): + """``Router.__init__`` registers every deployment's backend key through + ``register_model`` (``_register_deployment_in_model_cost``, shared-backend + path). For a deployment whose model is not in the cost map and that + carries no pricing or ``model_info``, that registration used to + materialize an empty entry — after which ``litellm.completion_cost`` for + the unmapped model silently returned 0.0 process-wide instead of raising, + even for calls that never touch the router. + """ + from litellm import Router + + backend_model: Final = "deepinfra/fake-unmapped-router-model-empty-reg" + stripped_key: Final = backend_model.split("/", 1)[1] + assert backend_model not in litellm.model_cost + + router: Final = Router( + model_list=[ # mutable-ok: Router constructor payload + { # mutable-ok: deployment entry under test + "model_name": "my-unpriced-model", + "litellm_params": {"model": backend_model, "api_key": "fake-key"}, # mutable-ok: deployment params + } + ] + ) + + try: + assert backend_model not in litellm.model_cost, ( + "Router.__init__ materialized an empty model_cost entry for an " + "unmapped deployment with no pricing" + ) + with pytest.raises(Exception, match="isn't mapped"): + litellm.completion_cost( + model=backend_model, prompt="hi", completion="there" + ) + finally: + del router + litellm.model_cost.pop(backend_model, None) + litellm.model_cost.pop(stripped_key, None) + _invalidate_model_cost_lowercase_map() diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index ea8a105ef6c..a6ba52712ea 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1175,23 +1175,29 @@ def test_repointing_a_deployment_drops_its_previous_backend_key(monkeypatch): dict(litellm_utils._runtime_registered_model_cost), ) + # ``mode`` is a cost-map schema field, so it flows through to the shared + # backend key. Without at least one such field the shared payload is empty + # and ``register_model`` (correctly) registers nothing for the backend key, + # which would leave this test nothing to observe. router = Router( model_list=[ { "model_name": "moving-target", "litellm_params": {"model": "hosted_vllm/old-backend"}, - "model_info": {"id": "moving-target-id"}, + "model_info": {"id": "moving-target-id", "mode": "chat"}, } ], ) saved_model_cost = litellm.model_cost try: + assert "hosted_vllm/old-backend" in litellm.model_cost + router.upsert_deployment( deployment=Deployment( model_name="moving-target", litellm_params=LiteLLM_Params(model="hosted_vllm/new-backend"), - model_info=ModelInfo(id="moving-target-id"), + model_info=ModelInfo(id="moving-target-id", mode="chat"), ) ) From 4e931e09d3c477ab82c10de7c0b14447d650559a Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:39:59 +0530 Subject: [PATCH 2/3] chore: re-trigger CI after retargeting base to litellm_internal_staging Co-Authored-By: Claude Fable 5 From 3e80302f891dc8b1948b274f244745816416e6a7 Mon Sep 17 00:00:00 2001 From: Ambuj Upadhyay <34904987+lets-order-some-fries@users.noreply.github.com> Date: Thu, 13 Aug 2026 21:45:15 +0530 Subject: [PATCH 3/3] chore: drop explanatory source comment per repo comment policy Rationale lives in the PR body and the regression tests. Co-Authored-By: Claude Fable 5 --- litellm/utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9702e239f56..3c9f624721f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2807,12 +2807,6 @@ def register_model(model_cost: str | dict, *, persist_across_reloads: bool = Tru existing_model.pop(_cost_field, None) ## override / add new keys to the existing model cost dictionary updated_dictionary = _update_dictionary(existing_model, value) - # An empty payload for a model with no built-in entry carries no - # information — do not materialize a bare key for it. Key existence is - # what cost lookup treats as "mapped", so an empty entry flips - # ``completion_cost`` for an unmapped model from a loud "model isn't - # mapped" error to a silent $0.0 (``Router.__init__`` registers every - # deployment's backend key this way when no pricing is configured). if updated_dictionary: litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary)