scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix(cost): bill the requested model's rates when the response names an unpriced model

Browse source 
Fireworks answers chat and /v1/messages calls with a display name such as
"Qwen 3.8 Max" in the response's model field. The cost path priced that
name, the Fireworks calculator fell back to its free default bucket without
raising, and the call billed $0 even though the requested model has a row.

When the response model resolves to no cost-map row or rule that declares
rates and the requested model does, price from the requested model. A priced
response model still wins, so dated snapshots keep their own rates, and an
unpriced requested model such as a router alias leaves the result unchanged.
This commit is contained in:
mateo-berri 2026-10-28 05:53:52 -07:00
parent 93056ef939
commit fcb22cb202
2 changed files with 130 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
litellm/cost_calculator.py
View file

@ -961,19 +961,14 @@ def select_model_name_for_cost_calc(
"""
1. If custom pricing is true, return received model name
2. If base_model is set (e.g. for azure models), return that
3. If completion response has model set return that
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
4. Check if model is passed in return that
"""
return_model: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
if completion_response is not None:
if isinstance(completion_response, BaseModel):
completion_response_model = getattr(completion_response, "model", None)
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
completion_response_model: Final = _get_response_model(completion_response)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
explicit_pricing: Final = custom_pricing is True or base_model is not None
@ -1009,27 +1004,43 @@ def select_model_name_for_cost_calc(
return_model = hidden_params.get("model", model)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model
return _served_or_priced_requested_model(
served=_cost_map_model_name(completion_response_model, custom_llm_provider, priced_region),
requested=_cost_map_model_name(model, custom_llm_provider, None) if model is not None else None,
custom_llm_provider=custom_llm_provider,
)
if return_model is None and model is not None:
return_model = model
if (
return_model is not None
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
provider_prefix: Final = (
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
)
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
return return_model
if return_model is None:
return None
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
return model
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
if rule_info is not None:
return _cost_map_entry_prices_anything(rule_info)
return _declares_rates(_cost_map_model_info(model, custom_llm_provider))
def _served_or_priced_requested_model(served: str, requested: str | None, custom_llm_provider: str | None) -> str:
if requested is None or requested == served or _resolves_to_rates(served, custom_llm_provider):
return served
return requested if _resolves_to_rates(requested, custom_llm_provider) else served
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already

100
tests/unit/test_cost_calculator.py
View file

@ -424,6 +424,106 @@ def test_completion_cost_prices_rule_only_base_model_from_the_response_model(
assert cost == pytest.approx(usage.prompt_tokens * 0.0000111 + usage.completion_tokens * 0.0000222)
def _token_rates_cost(row: Mapping[str, object], usage: Usage) -> float:
input_rate: Final = row["input_cost_per_token"]
output_rate: Final = row["output_cost_per_token"]
assert isinstance(input_rate, float) and isinstance(output_rate, float), row
return usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate
@pytest.mark.parametrize(
"requested_model",
["accounts/fireworks/models/sol-display-test", "fireworks_ai/accounts/fireworks/models/sol-display-test"],
)
def test_completion_cost_bills_requested_model_when_response_names_an_unmapped_display_name(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, requested_model: str
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"fireworks_ai/accounts/fireworks/models/sol-display-test",
{"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6},
)
usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
cost: Final = completion_cost(
completion_response=ModelResponse(model="Sol Display Max", choices=[], usage=usage),
model=requested_model,
custom_llm_provider="fireworks_ai",
)
assert cost == pytest.approx(
_token_rates_cost(litellm.model_cost["fireworks_ai/accounts/fireworks/models/sol-display-test"], usage)
)
def test_completion_cost_bills_requested_model_when_response_row_has_no_rates(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"openai/sol-requested-test",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6},
)
monkeypatch.setitem(
litellm.model_cost,
"openai/sol-served-snapshot-test",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None},
)
usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
cost: Final = completion_cost(
completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage),
model="openai/sol-requested-test",
custom_llm_provider="openai",
)
assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test"], usage))
def test_completion_cost_bills_priced_response_model_over_priced_requested_model(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"openai/sol-requested-test",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6},
)
monkeypatch.setitem(
litellm.model_cost,
"openai/sol-requested-test-2099-01-01",
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6},
)
usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
cost: Final = completion_cost(
completion_response=ModelResponse(model="sol-requested-test-2099-01-01", choices=[], usage=usage),
model="openai/sol-requested-test",
custom_llm_provider="openai",
)
assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test-2099-01-01"], usage))
def test_completion_cost_keeps_response_pricing_when_requested_alias_has_no_rates(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
size_bucket: Final = get_base_model_for_pricing("Sol 70B Display")
monkeypatch.setitem(
litellm.model_cost,
size_bucket,
{"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6},
)
usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
cost: Final = completion_cost(
completion_response=ModelResponse(model="Sol 70B Display", choices=[], usage=usage),
model="sol-team-alias",
custom_llm_provider="fireworks_ai",
)
assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[size_bucket], usage))
def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests(
_local_model_cost_map: None,
) -> None: