mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(huggingface): respect registered model_cost for TEI rerank pricing
Skip the synthetic zero-cost huggingface ModelInfo fallback when the model is already present in litellm.model_cost, so input_cost_per_query on a TEI rerank deployment is billed instead of always returning $0. Fixes #45975
This commit is contained in:
parent
0bd677afef
commit
707e506333
2 changed files with 36 additions and 1 deletions
|
|
@ -7050,7 +7050,11 @@ def get_model_info_helper(
|
|||
e,
|
||||
)
|
||||
|
||||
if not catalog_only and custom_llm_provider == "huggingface":
|
||||
if (
|
||||
not catalog_only
|
||||
and custom_llm_provider == "huggingface"
|
||||
and not _is_potential_model_name_in_model_cost(potential_model_names)
|
||||
):
|
||||
max_tokens: Final = _get_max_position_embeddings(model_name=model)
|
||||
return ModelInfoBase(
|
||||
key=model,
|
||||
|
|
|
|||
|
|
@ -1898,6 +1898,37 @@ def test_tiered_pricing_only_deployment_completion_cost_is_nonzero():
|
|||
assert cost > 0
|
||||
|
||||
|
||||
def test_huggingface_rerank_uses_registered_input_cost_per_query():
|
||||
"""Registered per-query price must apply for huggingface/TEI rerank (#45975).
|
||||
|
||||
``_get_model_info_helper`` used to return a synthetic zero-cost huggingface
|
||||
description before consulting ``litellm.model_cost``, so a deployment priced
|
||||
only via ``input_cost_per_query`` always billed $0 under that provider.
|
||||
"""
|
||||
from litellm.cost_calculator import rerank_cost
|
||||
from litellm.utils import cached_get_model_info_helper
|
||||
|
||||
cached_get_model_info_helper.cache_clear()
|
||||
litellm.model_cost.pop("hf-tei-rerank-deployment", None)
|
||||
litellm.register_model({"hf-tei-rerank-deployment": {"input_cost_per_query": 0.001}})
|
||||
|
||||
cost = rerank_cost(
|
||||
model="hf-tei-rerank-deployment",
|
||||
custom_llm_provider="huggingface",
|
||||
billed_units={"search_units": 1},
|
||||
)
|
||||
assert cost == (0.001, 0.0)
|
||||
|
||||
# Unregistered huggingface models still get the synthetic zero-cost fallback.
|
||||
cached_get_model_info_helper.cache_clear()
|
||||
info = litellm.get_model_info(
|
||||
model="unregistered-hf-tei-model-xyz",
|
||||
custom_llm_provider="huggingface",
|
||||
)
|
||||
assert info["input_cost_per_token"] == 0
|
||||
assert info["output_cost_per_token"] == 0
|
||||
|
||||
|
||||
def test_per_query_priced_rerank_deployment_completion_cost_is_nonzero():
|
||||
"""A rerank deployment priced only via ``input_cost_per_query`` must resolve
|
||||
cost against its ``router_model_id`` entry: the shared backend alias has
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue