fix(rerank): use the provider-reported cost for rerank responses

A rerank provider that reports its own charge in usage.cost lost it in the
response transform, so a rerank missing from the cost map was priced at 0.0.
RerankBilledUnits now keeps cost, completion_cost passes it on unless the
deployment has custom pricing, and rerank_cost prefers it over the cost map

Fixes #47796
This commit is contained in:
Tannmay-ts 2026-10-21 20:22:41 +05:30
parent b6f91a221b
commit ac1cda4af9
4 changed files with 60 additions and 0 deletions

View file

@ -1874,6 +1874,7 @@ def completion_cost(
rerank_billed_units = RerankBilledUnits(
search_units=billed_units.get("search_units"),
total_tokens=billed_units.get("total_tokens"),
cost=None if custom_pricing else billed_units.get("cost"),
)
search_units = billed_units.get("search_units") or 1 # cohere charges per request by default.
@ -2671,6 +2672,10 @@ def rerank_cost(
Returns
- float or None: cost of response OR none if error.
"""
provider_reported_cost: Final = billed_units.get("cost") if billed_units is not None else None
if provider_reported_cost is not None:
return provider_reported_cost, 0.0
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
try:

View file

@ -56,6 +56,7 @@ class OptionalRerankParams(TypedDict, total=False):
class RerankBilledUnits(TypedDict, total=False):
search_units: int | None
total_tokens: int | None
cost: ReadOnly[float | None]
class RerankTokens(TypedDict, total=False):

View file

@ -48,6 +48,10 @@ def test_transform_rerank_response_maps_results_and_usage():
{"results": [], "usage": {"total_tokens": 12, "input_tokens": 7, "output_tokens": 3, "unknown": 1}},
{"billed_units": {"total_tokens": 12}, "tokens": {"input_tokens": 7, "output_tokens": 3}},
),
(
{"results": [], "usage": {"search_units": 1, "cost": 0.001}},
{"billed_units": {"search_units": 1, "cost": 0.001}, "tokens": {}},
),
],
)
def test_transform_rerank_response_keeps_only_known_usage_counters(

View file

@ -1599,6 +1599,56 @@ def test_per_query_priced_rerank_deployment_completion_cost_is_nonzero():
assert cost == pytest.approx(3 * 0.001)
def test_rerank_completion_cost_uses_the_provider_reported_cost():
model: Final = "jina_ai/reranker-missing-from-the-cost-map"
assert litellm.model_cost.get(model) is None
response: Final = RerankResponse(
id="provider_reported_rerank_cost",
results=[{"index": 0, "relevance_score": 0.9}],
meta={"billed_units": {"search_units": 1, "cost": 0.001}},
)
cost: Final = completion_cost(
completion_response=response, model=model, custom_llm_provider="jina_ai", call_type="arerank"
)
assert cost == pytest.approx(0.001)
def test_rerank_custom_pricing_wins_over_the_provider_reported_cost():
from litellm import Router
router: Final = Router(
model_list=[
{
"model_name": "semantic-ranker-reported-cost-test",
"litellm_params": {
"model": "vertex_ai/semantic-ranker-reported-cost-test",
"vertex_project": "test-project",
"vertex_location": "us-east5",
},
"model_info": {"input_cost_per_query": 0.002},
},
]
)
response: Final = RerankResponse(
id="custom_priced_rerank_with_reported_cost",
results=[{"index": 0, "relevance_score": 0.9}],
meta={"billed_units": {"search_units": 3, "cost": 0.5}},
)
cost: Final = completion_cost(
completion_response=response,
model="vertex_ai/semantic-ranker-reported-cost-test",
custom_llm_provider="vertex_ai",
call_type="arerank",
custom_pricing=True,
router_model_id=router.model_list[0]["model_info"]["id"],
)
assert cost == pytest.approx(3 * 0.002)
@pytest.mark.parametrize(
"model",
(