mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(utils): stop register_model materializing empty model_cost entries
register_model() ran litellm.model_cost.setdefault(key, {}) even when the
merged payload was empty, so registering an unmapped model with an empty
payload created a bare key. Key existence is what cost lookup treats as
'mapped', so completion_cost() for that model flipped from raising 'This
model isn't mapped yet' to silently returning $0.0 — process-wide.
Router.__init__ hits this for every deployment: the shared-backend
registration (_register_deployment_in_model_cost) strips per-deployment
pricing/metadata from the shared payload, so any deployment configured
without pricing registers {} under its backend model name.
Fix: only write to litellm.model_cost when the merged payload is
non-empty.
Test updates, both verified against pre-fix behavior:
- test_register_model_router_add_deployment_custom_pricing_applies
asserted on the backend keys, but pre-fix those held exactly the empty
entry ({}), making _check_provider_match vacuously True; the custom
pricing lives (and always lived) under the deployment id entry. The
test now asserts on that entry.
- test_repointing_a_deployment_drops_its_previous_backend_key /
test_tiered_pricing_only_deployment_selects_router_model_id relied on
the empty shared entry existing; the deployments now carry (or
tolerate the absence of) a real shared field so the guarantees stay
exercised.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
92d453373a
commit
24d0f029c4
4 changed files with 98 additions and 10 deletions
|
|
@ -3119,12 +3119,19 @@ def register_model(
|
|||
existing_model.pop(_cost_field, None)
|
||||
## override / add new keys to the existing model cost dictionary
|
||||
updated_dictionary = _update_dictionary(existing_model, value)
|
||||
litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary)
|
||||
# An empty payload for a model with no built-in entry carries no
|
||||
# information — do not materialize a bare key for it. Key existence is
|
||||
# what cost lookup treats as "mapped", so an empty entry flips
|
||||
# ``completion_cost`` for an unmapped model from a loud "model isn't
|
||||
# mapped" error to a silent $0.0 (``Router.__init__`` registers every
|
||||
# deployment's backend key this way when no pricing is configured).
|
||||
if updated_dictionary:
|
||||
litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary)
|
||||
|
||||
# Invalidate case-insensitive lookup map since model_cost was modified
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
# Invalidate case-insensitive lookup map since model_cost was modified
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key)
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", model_cost_key, model_cost_key)
|
||||
# add new model names to provider lists
|
||||
if value.get("litellm_provider") == "openai":
|
||||
if key not in litellm.open_ai_chat_completion_models:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -1072,10 +1075,11 @@ def test_tiered_pricing_only_deployment_selects_router_model_id():
|
|||
entry = litellm.model_cost[router_model_id]
|
||||
assert entry.get("input_cost_per_token") is None
|
||||
assert entry.get("tiered_pricing") is not None
|
||||
# The stripped shared alias must not carry tiered pricing.
|
||||
assert (
|
||||
litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None
|
||||
)
|
||||
# The stripped shared alias must not carry tiered pricing. After
|
||||
# stripping, this deployment contributes nothing to the shared key, so
|
||||
# ``register_model`` may not have materialized it at all.
|
||||
shared_alias_entry: Final = litellm.model_cost.get("dashscope/qwen-tier-only-test")
|
||||
assert shared_alias_entry is None or shared_alias_entry.get("tiered_pricing") is None
|
||||
|
||||
selected = _select_model_name_for_cost_calc(
|
||||
model="dashscope/qwen-tier-only-test",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ calculations for DB-sourced models with prompt caching pricing.
|
|||
|
||||
import copy
|
||||
import os
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -993,3 +995,72 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
|
|||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
del router
|
||||
|
||||
|
||||
def test_register_model_empty_payload_does_not_materialize_entry():
|
||||
"""An empty payload for a model with no built-in entry carries no
|
||||
information, and ``register_model`` must not create a bare key for it in
|
||||
``litellm.model_cost``. Key existence is what cost lookup treats as
|
||||
"mapped", so an empty entry flips ``completion_cost`` for an unmapped
|
||||
model from a loud "model isn't mapped" error to a silent $0.0.
|
||||
"""
|
||||
from litellm.utils import _runtime_registered_model_cost
|
||||
|
||||
model_key: Final = "deepinfra/fake-unmapped-model-empty-registration"
|
||||
assert model_key not in litellm.model_cost
|
||||
with pytest.raises(Exception, match="isn't mapped"):
|
||||
litellm.completion_cost(model=model_key, prompt="hi", completion="there")
|
||||
|
||||
try:
|
||||
litellm.register_model({model_key: {}}) # mutable-ok: registration payload under test
|
||||
|
||||
assert model_key not in litellm.model_cost, (
|
||||
"register_model materialized an empty model_cost entry for an "
|
||||
"unmapped model registered with an empty payload"
|
||||
)
|
||||
with pytest.raises(Exception, match="isn't mapped"):
|
||||
litellm.completion_cost(model=model_key, prompt="hi", completion="there")
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_runtime_registered_model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_router_init_without_pricing_keeps_unmapped_model_cost_loud():
|
||||
"""``Router.__init__`` registers every deployment's backend key through
|
||||
``register_model`` (``_register_deployment_in_model_cost``, shared-backend
|
||||
path). For a deployment whose model is not in the cost map and that
|
||||
carries no pricing or ``model_info``, that registration used to
|
||||
materialize an empty entry — after which ``litellm.completion_cost`` for
|
||||
the unmapped model silently returned 0.0 process-wide instead of raising,
|
||||
even for calls that never touch the router.
|
||||
"""
|
||||
from litellm import Router
|
||||
|
||||
backend_model: Final = "deepinfra/fake-unmapped-router-model-empty-reg"
|
||||
stripped_key: Final = backend_model.split("/", 1)[1]
|
||||
assert backend_model not in litellm.model_cost
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[ # mutable-ok: Router constructor payload
|
||||
{ # mutable-ok: deployment entry under test
|
||||
"model_name": "my-unpriced-model",
|
||||
"litellm_params": {"model": backend_model, "api_key": "fake-key"}, # mutable-ok: deployment params
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
try:
|
||||
assert backend_model not in litellm.model_cost, (
|
||||
"Router.__init__ materialized an empty model_cost entry for an "
|
||||
"unmapped deployment with no pricing"
|
||||
)
|
||||
with pytest.raises(Exception, match="isn't mapped"):
|
||||
litellm.completion_cost(
|
||||
model=backend_model, prompt="hi", completion="there"
|
||||
)
|
||||
finally:
|
||||
del router
|
||||
litellm.model_cost.pop(backend_model, None)
|
||||
litellm.model_cost.pop(stripped_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
|
|
|||
|
|
@ -1282,23 +1282,29 @@ def test_repointing_a_deployment_drops_its_previous_backend_key(monkeypatch):
|
|||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
# ``mode`` is a cost-map schema field, so it flows through to the shared
|
||||
# backend key. Without at least one such field the shared payload is empty
|
||||
# and ``register_model`` (correctly) registers nothing for the backend key,
|
||||
# which would leave this test nothing to observe.
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "moving-target",
|
||||
"litellm_params": {"model": "hosted_vllm/old-backend"},
|
||||
"model_info": {"id": "moving-target-id"},
|
||||
"model_info": {"id": "moving-target-id", "mode": "chat"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
saved_model_cost = litellm.model_cost
|
||||
try:
|
||||
assert "hosted_vllm/old-backend" in litellm.model_cost
|
||||
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="moving-target",
|
||||
litellm_params=LiteLLM_Params(model="hosted_vllm/new-backend"),
|
||||
model_info=ModelInfo(id="moving-target-id"),
|
||||
model_info=ModelInfo(id="moving-target-id", mode="chat"),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue