mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(cost): try exactly priced model names before capability-rule matches in completion_cost
The proxy restamps streaming chunks' model field to the client-requested model group name, so the reassembled response used for cost calculation carries the alias instead of the deployment model. Since the fallback generalizations feature, an unmapped alias that prefix-matches a Claude capability rule (e.g. anthropic/claude-sonnet-4-6-defaultcaching) no longer raises on lookup; it resolves to a rule-derived entry whose synthesized pricing is 0, so the completion_cost candidate ladder stops there and bills the request at $0 even though a later candidate (the deployment's real model name) has exact pricing. Order the ladder so candidates that resolve only via capability rules are tried last, extending the existing rules-lose-to-exact-entries invariant across the ladder. Models with no exactly priced candidate at all still resolve through their rule as before.
This commit is contained in:
parent
5d4c4d0fce
commit
fbac372651
2 changed files with 95 additions and 4 deletions
|
|
@ -799,6 +799,36 @@ def _select_model_name_for_cost_calc(
|
|||
return return_model
|
||||
|
||||
|
||||
def _prices_only_via_capability_rule(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
"""Return True when ``model`` has no exact cost-map entry under any of its
|
||||
name variants and resolves only through a capability generalization rule.
|
||||
|
||||
Rule-derived entries describe capabilities but carry no pricing, so
|
||||
``get_model_info`` synthesizes them with input/output cost 0. If the cost
|
||||
ladder accepted such a candidate as a successful pricing lookup, the
|
||||
request would be billed $0 even though a later candidate (e.g. the
|
||||
deployment's real model name, when the response carries a router
|
||||
model-group alias stamped on streaming chunks) has exact pricing. The
|
||||
ladder therefore tries exactly-priced candidates first and rule-derived
|
||||
ones last.
|
||||
"""
|
||||
if model is None:
|
||||
return False
|
||||
from litellm.utils import (
|
||||
_get_model_info_from_generalization,
|
||||
_get_potential_model_names,
|
||||
)
|
||||
|
||||
return (
|
||||
_get_model_info_from_generalization(
|
||||
model=model,
|
||||
potential_model_names=_get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
|
||||
def _model_contains_known_llm_provider(model: str) -> bool:
|
||||
"""
|
||||
|
|
@ -1218,12 +1248,16 @@ def completion_cost(
|
|||
router_model_id=router_model_id,
|
||||
)
|
||||
|
||||
potential_model_names = [
|
||||
_candidate_model_names = (
|
||||
selected_model,
|
||||
_get_response_model(completion_response),
|
||||
]
|
||||
if model is not None:
|
||||
potential_model_names.append(model)
|
||||
) + ((model,) if model is not None else ())
|
||||
potential_model_names = sorted(
|
||||
_candidate_model_names,
|
||||
key=lambda candidate: _prices_only_via_capability_rule(
|
||||
model=candidate, custom_llm_provider=custom_llm_provider
|
||||
),
|
||||
)
|
||||
|
||||
for idx, model in enumerate(potential_model_names):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -3479,3 +3479,60 @@ def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate():
|
|||
)
|
||||
|
||||
assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2)
|
||||
|
||||
|
||||
def test_streaming_alias_response_model_loses_to_exactly_priced_candidate(monkeypatch):
|
||||
"""A router model-group alias (e.g. ``anthropic/claude-sonnet-4-6-defaultcaching``)
|
||||
is stamped onto streaming chunks by the proxy, so the reassembled response
|
||||
carries the alias as its model name. The alias has no cost-map entry but
|
||||
prefix-matches a Claude capability generalization rule, whose synthesized
|
||||
entry prices at $0. The cost ladder must prefer the exactly priced
|
||||
deployment model over the rule-derived alias instead of billing $0.
|
||||
Regression test for streaming requests on Claude-named alias groups being
|
||||
logged with zero spend since the fallback-generalizations feature.
|
||||
"""
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
match_capability_generalizations,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
alias = "anthropic/claude-sonnet-4-6-defaultcaching"
|
||||
assert alias not in litellm.model_cost
|
||||
assert match_capability_generalizations("claude-sonnet-4-6-defaultcaching") is not None
|
||||
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=10)
|
||||
alias_cost = completion_cost(
|
||||
completion_response=ModelResponse(usage=usage, model=alias),
|
||||
model="anthropic/claude-sonnet-4-6",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
real_cost = completion_cost(
|
||||
completion_response=ModelResponse(usage=usage, model="claude-sonnet-4-6"),
|
||||
model="anthropic/claude-sonnet-4-6",
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert real_cost > 0
|
||||
assert alias_cost == real_cost
|
||||
|
||||
|
||||
def test_capability_rule_still_prices_model_with_no_exactly_priced_candidate(monkeypatch):
|
||||
"""When every ladder candidate is unmapped and only a capability rule
|
||||
matches, the rule-derived entry must still resolve (at its synthesized $0)
|
||||
rather than raising, preserving the fallback-generalizations behavior for
|
||||
genuinely unknown models.
|
||||
"""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
alias = "anthropic/claude-sonnet-4-6-defaultcaching"
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=10)
|
||||
cost = completion_cost(
|
||||
completion_response=ModelResponse(usage=usage, model="claude-sonnet-4-6-defaultcaching"),
|
||||
model=alias,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue