mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(rerank): use the provider-reported cost for rerank responses
A rerank provider that reports its own charge in usage.cost lost it in the response transform, so a rerank missing from the cost map was priced at 0.0. RerankBilledUnits now keeps cost, completion_cost passes it on unless the deployment has custom pricing, and rerank_cost prefers it over the cost map Fixes #47796
This commit is contained in:
parent
b6f91a221b
commit
ac1cda4af9
4 changed files with 60 additions and 0 deletions
|
|
@ -1874,6 +1874,7 @@ def completion_cost(
|
|||
rerank_billed_units = RerankBilledUnits(
|
||||
search_units=billed_units.get("search_units"),
|
||||
total_tokens=billed_units.get("total_tokens"),
|
||||
cost=None if custom_pricing else billed_units.get("cost"),
|
||||
)
|
||||
|
||||
search_units = billed_units.get("search_units") or 1 # cohere charges per request by default.
|
||||
|
|
@ -2671,6 +2672,10 @@ def rerank_cost(
|
|||
Returns
|
||||
- float or None: cost of response OR none if error.
|
||||
"""
|
||||
provider_reported_cost: Final = billed_units.get("cost") if billed_units is not None else None
|
||||
if provider_reported_cost is not None:
|
||||
return provider_reported_cost, 0.0
|
||||
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ class OptionalRerankParams(TypedDict, total=False):
|
|||
class RerankBilledUnits(TypedDict, total=False):
|
||||
search_units: int | None
|
||||
total_tokens: int | None
|
||||
cost: ReadOnly[float | None]
|
||||
|
||||
|
||||
class RerankTokens(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ def test_transform_rerank_response_maps_results_and_usage():
|
|||
{"results": [], "usage": {"total_tokens": 12, "input_tokens": 7, "output_tokens": 3, "unknown": 1}},
|
||||
{"billed_units": {"total_tokens": 12}, "tokens": {"input_tokens": 7, "output_tokens": 3}},
|
||||
),
|
||||
(
|
||||
{"results": [], "usage": {"search_units": 1, "cost": 0.001}},
|
||||
{"billed_units": {"search_units": 1, "cost": 0.001}, "tokens": {}},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_transform_rerank_response_keeps_only_known_usage_counters(
|
||||
|
|
|
|||
|
|
@ -1599,6 +1599,56 @@ def test_per_query_priced_rerank_deployment_completion_cost_is_nonzero():
|
|||
assert cost == pytest.approx(3 * 0.001)
|
||||
|
||||
|
||||
def test_rerank_completion_cost_uses_the_provider_reported_cost():
|
||||
model: Final = "jina_ai/reranker-missing-from-the-cost-map"
|
||||
assert litellm.model_cost.get(model) is None
|
||||
response: Final = RerankResponse(
|
||||
id="provider_reported_rerank_cost",
|
||||
results=[{"index": 0, "relevance_score": 0.9}],
|
||||
meta={"billed_units": {"search_units": 1, "cost": 0.001}},
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response, model=model, custom_llm_provider="jina_ai", call_type="arerank"
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(0.001)
|
||||
|
||||
|
||||
def test_rerank_custom_pricing_wins_over_the_provider_reported_cost():
|
||||
from litellm import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "semantic-ranker-reported-cost-test",
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/semantic-ranker-reported-cost-test",
|
||||
"vertex_project": "test-project",
|
||||
"vertex_location": "us-east5",
|
||||
},
|
||||
"model_info": {"input_cost_per_query": 0.002},
|
||||
},
|
||||
]
|
||||
)
|
||||
response: Final = RerankResponse(
|
||||
id="custom_priced_rerank_with_reported_cost",
|
||||
results=[{"index": 0, "relevance_score": 0.9}],
|
||||
meta={"billed_units": {"search_units": 3, "cost": 0.5}},
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response,
|
||||
model="vertex_ai/semantic-ranker-reported-cost-test",
|
||||
custom_llm_provider="vertex_ai",
|
||||
call_type="arerank",
|
||||
custom_pricing=True,
|
||||
router_model_id=router.model_list[0]["model_info"]["id"],
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(3 * 0.002)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue