mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(cost_calculator): price base_model deployments from the base model's own provider
A hosted_vllm deployment with model_info.base_model set to a registered cost key from a different provider (e.g. openrouter/deepseek/...) recorded /usr/bin/bash spend: cost_per_token received the deployment's provider and _get_model_info_helper rejected the base model's entry on provider mismatch, so every candidate in completion_cost failed and cost fell back to 0.0. A bare base_model also broke: it was prefixed with the deployment provider into an unregistered key before pricing. When base_model is set, infer custom_llm_provider from the selected model itself so pricing follows the base model's own provider entry, and keep a bare base_model verbatim when the prefixed form is unregistered. Fixes #41780
This commit is contained in:
parent
8fc9c46d1a
commit
1e181767ce
2 changed files with 54 additions and 2 deletions
|
|
@ -844,7 +844,11 @@ def _select_model_name_for_cost_calc(
|
|||
provider_prefix: Final = (
|
||||
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
|
||||
)
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
prefixed_model: Final = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
if not (
|
||||
base_model is not None and prefixed_model not in litellm.model_cost and return_model in litellm.model_cost
|
||||
): # base_model is an explicit cost key; keep it verbatim when the prefixed form is unregistered (#41780)
|
||||
return_model = prefixed_model
|
||||
|
||||
return return_model
|
||||
|
||||
|
|
@ -1716,7 +1720,7 @@ def completion_cost(
|
|||
model=model,
|
||||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_llm_provider=None if base_model is not None else custom_llm_provider,
|
||||
response_time_ms=total_time,
|
||||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
|
|
|
|||
|
|
@ -3537,6 +3537,54 @@ def test_completion_cost_base_model_ignores_regional_row(_local_model_cost_map):
|
|||
) == pytest.approx(1000 * flat["input_cost_per_token"])
|
||||
|
||||
|
||||
def test_completion_cost_base_model_cross_provider_prefix(_local_model_cost_map):
|
||||
"""base_model pointing at a registered cost key from a different provider than the deployment
|
||||
is priced from that key instead of failing provider matching and recording $0 spend (#41780)."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="custom-model",
|
||||
usage={"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100},
|
||||
)
|
||||
response._hidden_params = {"custom_llm_provider": "hosted_vllm"}
|
||||
entry = litellm.model_cost["openrouter/deepseek/deepseek-v4-flash-0731"]
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="custom-model",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
base_model="openrouter/deepseek/deepseek-v4-flash-0731",
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(1000 * entry["input_cost_per_token"] + 100 * entry["output_cost_per_token"])
|
||||
|
||||
|
||||
def test_completion_cost_base_model_bare_name_kept_when_prefix_unregistered(_local_model_cost_map):
|
||||
"""A bare base_model whose deployment-prefixed form is not a registered cost key is priced
|
||||
from the base model's own registered key instead of being prefixed into an unresolvable
|
||||
name and recording $0 spend (#41780)."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model="custom-model",
|
||||
usage={"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100},
|
||||
)
|
||||
response._hidden_params = {"custom_llm_provider": "hosted_vllm"}
|
||||
assert "hosted_vllm/gpt-3.5-turbo" not in litellm.model_cost
|
||||
entry = litellm.model_cost["gpt-3.5-turbo"]
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="custom-model",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
base_model="gpt-3.5-turbo",
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(1000 * entry["input_cost_per_token"] + 100 * entry["output_cost_per_token"])
|
||||
|
||||
|
||||
def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map):
|
||||
"""An alias that resolves to no known cost key keeps the legacy double-prefixed name."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue