diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..712b57435cb 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1874,6 +1874,7 @@ def completion_cost( rerank_billed_units = RerankBilledUnits( search_units=billed_units.get("search_units"), total_tokens=billed_units.get("total_tokens"), + cost=None if custom_pricing else billed_units.get("cost"), ) search_units = billed_units.get("search_units") or 1 # cohere charges per request by default. @@ -2671,6 +2672,10 @@ def rerank_cost( Returns - float or None: cost of response OR none if error. """ + provider_reported_cost: Final = billed_units.get("cost") if billed_units is not None else None + if provider_reported_cost is not None: + return provider_reported_cost, 0.0 + _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) try: diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index 6d1ab85012b..8032caa8764 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -56,6 +56,7 @@ class OptionalRerankParams(TypedDict, total=False): class RerankBilledUnits(TypedDict, total=False): search_units: int | None total_tokens: int | None + cost: ReadOnly[float | None] class RerankTokens(TypedDict, total=False): diff --git a/tests/unit/llms/jina_ai/rerank/test_transformation.py b/tests/unit/llms/jina_ai/rerank/test_transformation.py index 9ec02a4548b..e4b5aad9c05 100644 --- a/tests/unit/llms/jina_ai/rerank/test_transformation.py +++ b/tests/unit/llms/jina_ai/rerank/test_transformation.py @@ -48,6 +48,10 @@ def test_transform_rerank_response_maps_results_and_usage(): {"results": [], "usage": {"total_tokens": 12, "input_tokens": 7, "output_tokens": 3, "unknown": 1}}, {"billed_units": {"total_tokens": 12}, "tokens": {"input_tokens": 7, "output_tokens": 3}}, ), + ( + {"results": [], "usage": {"search_units": 1, "cost": 0.001}}, + {"billed_units": {"search_units": 1, "cost": 0.001}, "tokens": {}}, + ), ], ) def test_transform_rerank_response_keeps_only_known_usage_counters( diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 5d4eb5fae12..8fa731b804a 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1599,6 +1599,56 @@ def test_per_query_priced_rerank_deployment_completion_cost_is_nonzero(): assert cost == pytest.approx(3 * 0.001) +def test_rerank_completion_cost_uses_the_provider_reported_cost(): + model: Final = "jina_ai/reranker-missing-from-the-cost-map" + assert litellm.model_cost.get(model) is None + response: Final = RerankResponse( + id="provider_reported_rerank_cost", + results=[{"index": 0, "relevance_score": 0.9}], + meta={"billed_units": {"search_units": 1, "cost": 0.001}}, + ) + + cost: Final = completion_cost( + completion_response=response, model=model, custom_llm_provider="jina_ai", call_type="arerank" + ) + + assert cost == pytest.approx(0.001) + + +def test_rerank_custom_pricing_wins_over_the_provider_reported_cost(): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "semantic-ranker-reported-cost-test", + "litellm_params": { + "model": "vertex_ai/semantic-ranker-reported-cost-test", + "vertex_project": "test-project", + "vertex_location": "us-east5", + }, + "model_info": {"input_cost_per_query": 0.002}, + }, + ] + ) + response: Final = RerankResponse( + id="custom_priced_rerank_with_reported_cost", + results=[{"index": 0, "relevance_score": 0.9}], + meta={"billed_units": {"search_units": 3, "cost": 0.5}}, + ) + + cost: Final = completion_cost( + completion_response=response, + model="vertex_ai/semantic-ranker-reported-cost-test", + custom_llm_provider="vertex_ai", + call_type="arerank", + custom_pricing=True, + router_model_id=router.model_list[0]["model_info"]["id"], + ) + + assert cost == pytest.approx(3 * 0.002) + + @pytest.mark.parametrize( "model", (