fix(cost): try exactly priced model names before capability-rule matches in completion_cost

The proxy restamps streaming chunks' model field to the client-requested
model group name, so the reassembled response used for cost calculation
carries the alias instead of the deployment model. Since the fallback
generalizations feature, an unmapped alias that prefix-matches a Claude
capability rule (e.g. anthropic/claude-sonnet-4-6-defaultcaching) no
longer raises on lookup; it resolves to a rule-derived entry whose
synthesized pricing is 0, so the completion_cost candidate ladder stops
there and bills the request at $0 even though a later candidate (the
deployment's real model name) has exact pricing.

Order the ladder so candidates that resolve only via capability rules
are tried last, extending the existing rules-lose-to-exact-entries
invariant across the ladder. Models with no exactly priced candidate at
all still resolve through their rule as before.
This commit is contained in:
Thomas Rehn 2026-07-22 11:00:22 +02:00
parent 5d4c4d0fce
commit fbac372651
2 changed files with 95 additions and 4 deletions

View file

@ -799,6 +799,36 @@ def _select_model_name_for_cost_calc(
return return_model
def _prices_only_via_capability_rule(model: str | None, custom_llm_provider: str | None) -> bool:
"""Return True when ``model`` has no exact cost-map entry under any of its
name variants and resolves only through a capability generalization rule.
Rule-derived entries describe capabilities but carry no pricing, so
``get_model_info`` synthesizes them with input/output cost 0. If the cost
ladder accepted such a candidate as a successful pricing lookup, the
request would be billed $0 even though a later candidate (e.g. the
deployment's real model name, when the response carries a router
model-group alias stamped on streaming chunks) has exact pricing. The
ladder therefore tries exactly-priced candidates first and rule-derived
ones last.
"""
if model is None:
return False
from litellm.utils import (
_get_model_info_from_generalization,
_get_potential_model_names,
)
return (
_get_model_info_from_generalization(
model=model,
potential_model_names=_get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider),
custom_llm_provider=custom_llm_provider,
)
is not None
)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _model_contains_known_llm_provider(model: str) -> bool:
"""
@ -1218,12 +1248,16 @@ def completion_cost(
router_model_id=router_model_id,
)
potential_model_names = [
_candidate_model_names = (
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
) + ((model,) if model is not None else ())
potential_model_names = sorted(
_candidate_model_names,
key=lambda candidate: _prices_only_via_capability_rule(
model=candidate, custom_llm_provider=custom_llm_provider
),
)
for idx, model in enumerate(potential_model_names):
try:

View file

@ -3479,3 +3479,60 @@ def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate():
)
assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2)
def test_streaming_alias_response_model_loses_to_exactly_priced_candidate(monkeypatch):
"""A router model-group alias (e.g. ``anthropic/claude-sonnet-4-6-defaultcaching``)
is stamped onto streaming chunks by the proxy, so the reassembled response
carries the alias as its model name. The alias has no cost-map entry but
prefix-matches a Claude capability generalization rule, whose synthesized
entry prices at $0. The cost ladder must prefer the exactly priced
deployment model over the rule-derived alias instead of billing $0.
Regression test for streaming requests on Claude-named alias groups being
logged with zero spend since the fallback-generalizations feature.
"""
from litellm.litellm_core_utils.fallback_generalizations import (
match_capability_generalizations,
)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
alias = "anthropic/claude-sonnet-4-6-defaultcaching"
assert alias not in litellm.model_cost
assert match_capability_generalizations("claude-sonnet-4-6-defaultcaching") is not None
usage = Usage(prompt_tokens=100, completion_tokens=10)
alias_cost = completion_cost(
completion_response=ModelResponse(usage=usage, model=alias),
model="anthropic/claude-sonnet-4-6",
custom_llm_provider="anthropic",
)
real_cost = completion_cost(
completion_response=ModelResponse(usage=usage, model="claude-sonnet-4-6"),
model="anthropic/claude-sonnet-4-6",
custom_llm_provider="anthropic",
)
assert real_cost > 0
assert alias_cost == real_cost
def test_capability_rule_still_prices_model_with_no_exactly_priced_candidate(monkeypatch):
"""When every ladder candidate is unmapped and only a capability rule
matches, the rule-derived entry must still resolve (at its synthesized $0)
rather than raising, preserving the fallback-generalizations behavior for
genuinely unknown models.
"""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
alias = "anthropic/claude-sonnet-4-6-defaultcaching"
usage = Usage(prompt_tokens=100, completion_tokens=10)
cost = completion_cost(
completion_response=ModelResponse(usage=usage, model="claude-sonnet-4-6-defaultcaching"),
model=alias,
custom_llm_provider="anthropic",
)
assert cost == 0.0