diff --git a/litellm/llms/bedrock/rerank/transformation.py b/litellm/llms/bedrock/rerank/transformation.py index 442903fc05e..61cba0d4d65 100644 --- a/litellm/llms/bedrock/rerank/transformation.py +++ b/litellm/llms/bedrock/rerank/transformation.py @@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works from typing import Final +from pydantic import TypeAdapter + from litellm._uuid import uuid from litellm.types.llms.bedrock import ( BedrockRerankBedrockRerankingConfiguration, @@ -27,6 +29,8 @@ from litellm.types.rerank import ( RerankTokens, ) +_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits) + class BedrockRerankConfig: def _transform_sources(self, documents: list[str | dict]) -> list[BedrockRerankSource]: @@ -90,7 +94,7 @@ class BedrockRerankConfig: example input: {"results":[{"index":0,"relevanceScore":0.6847912669181824},{"index":1,"relevanceScore":0.5980774760246277}]} """ - _billed_units = RerankBilledUnits(**response.get("usage", {"search_units": 1})) # by default 1 search unit + _billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {"search_units": 1})) _tokens: Final = RerankTokens(**response.get("usage", {})) rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens) diff --git a/litellm/llms/together_ai/rerank/transformation.py b/litellm/llms/together_ai/rerank/transformation.py index f1c3ca9e87d..ed15280a12a 100644 --- a/litellm/llms/together_ai/rerank/transformation.py +++ b/litellm/llms/together_ai/rerank/transformation.py @@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works from typing import Final +from pydantic import TypeAdapter + from litellm._uuid import uuid from litellm.types.rerank import ( RerankBilledUnits, @@ -16,10 +18,12 @@ from litellm.types.rerank import ( RerankTokens, ) +_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits) + class TogetherAIRerankConfig: def _transform_response(self, response: dict) -> RerankResponse: - _billed_units: Final = RerankBilledUnits(**response.get("usage", {})) + _billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {})) _tokens: Final = RerankTokens(**response.get("usage", {})) rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens) diff --git a/tests/unit/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py b/tests/unit/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py index 495e0f56c84..88ecdbfe05c 100644 --- a/tests/unit/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py +++ b/tests/unit/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py @@ -6,6 +6,8 @@ forward_client_headers_to_llm_api were not being passed to Bedrock rerank provid """ import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -15,7 +17,9 @@ from pydantic import ValidationError import litellm from litellm.llms.bedrock.base_aws_llm import Boto3CredentialsInfo from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler +from litellm.llms.bedrock.rerank.transformation import BedrockRerankConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.rerank import RerankBilledUnits @pytest.fixture(autouse=True) @@ -565,3 +569,37 @@ async def test_bedrock_arerank_rejects_non_object_bodies(payload: object): await _rerank_through_transport(payload, is_async=True) assert "sensitive-document" not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("usage_fields", "expected_billed_units"), + ( + pytest.param({}, {"search_units": 1}, id="missing-usage"), + pytest.param({"usage": {}}, {}, id="empty-usage"), + pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"), + pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"), + pytest.param( + { + "usage": { + "search_units": 2, + "total_tokens": 7, + "input_tokens": 5, + "output_tokens": 2, + "cost": 0.001, + "extra": 9, + } + }, + {"search_units": 2, "total_tokens": 7, "cost": 0.001}, + id="charged-with-token-counts", + ), + ), +) +def test_bedrock_rerank_preserves_billed_units( + usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits +) -> None: + response: Final = BedrockRerankConfig().transform_response( + {"results": [{"index": 0, "relevanceScore": 0.9}], **usage_fields} + ) + + assert response.meta is not None + assert response.meta["billed_units"] == expected_billed_units diff --git a/tests/unit/llms/scaleway/test_scaleway_rerank_transformation.py b/tests/unit/llms/scaleway/test_scaleway_rerank_transformation.py index dd448a048c6..140c4810475 100644 --- a/tests/unit/llms/scaleway/test_scaleway_rerank_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_rerank_transformation.py @@ -1,4 +1,5 @@ import json +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -7,6 +8,7 @@ import respx import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.rerank import RerankBilledUnits, RerankResponse SCALEWAY_RERANK_BODY = { "id": "rerank-a89e6d7b8b97492ea81569c65fbfff49", @@ -134,3 +136,72 @@ async def test_scaleway_arerank_posts_to_the_documented_endpoint(): assert client.post.await_args.kwargs["url"] == "https://api.scaleway.ai/v1/rerank" assert client.post.await_args.kwargs["headers"]["authorization"] == "Bearer scw-key" assert [r["index"] for r in response.results] == [1, 0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False), ids=("sync", "async")) +@pytest.mark.parametrize( + ("billed_units", "reported_cost"), + ( + pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.001}, 0.001, id="charged"), + pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.0}, 0.0, id="free"), + pytest.param({"search_units": 2, "total_tokens": 2}, None, id="absent-cost"), + pytest.param({"search_units": 2, "total_tokens": 2, "cost": None}, None, id="null-cost"), + ), +) +async def test_scaleway_rerank_preserves_openrouter_reported_cost( + sync_mode: bool, + billed_units: RerankBilledUnits, + reported_cost: float | None, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(litellm, "cache", None) + price_per_token: Final = 0.002 + monkeypatch.setitem( + litellm.model_cost, + "scaleway/cohere/rerank-v3.5", + {"input_cost_per_token": price_per_token, "litellm_provider": "scaleway", "mode": "rerank"}, + ) + route: Final = respx_mock.post("https://openrouter.ai/api/v1/rerank") + route.return_value = httpx.Response( + 200, + json={ + "id": "openrouter-rerank-cost-test", + "results": [{"index": 0, "relevance_score": 0.9}], + "usage": billed_units, + }, + ) + + response: Final = ( + litellm.rerank( + model="scaleway/cohere/rerank-v3.5", + api_base="https://openrouter.ai/api/v1", + api_key="test-openrouter-key", + query="the customer disputed the deployment terms", + documents=["the customer disputed the deployment terms"], + top_n=1, + ) + if sync_mode + else await litellm.arerank( + model="scaleway/cohere/rerank-v3.5", + api_base="https://openrouter.ai/api/v1", + api_key="test-openrouter-key", + query="the customer disputed the deployment terms", + documents=["the customer disputed the deployment terms"], + top_n=1, + ) + ) + + assert isinstance(response, RerankResponse) + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "cohere/rerank-v3.5", + "query": "the customer disputed the deployment terms", + "documents": ["the customer disputed the deployment terms"], + "top_n": 1, + } + assert response.meta == {"billed_units": billed_units, "tokens": {}} + expected_cost: Final = 2 * price_per_token if reported_cost is None else reported_cost + assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) diff --git a/tests/unit/llms/together_ai/rerank/test_handler.py b/tests/unit/llms/together_ai/rerank/test_handler.py index bc2f30364d2..2a26399731e 100644 --- a/tests/unit/llms/together_ai/rerank/test_handler.py +++ b/tests/unit/llms/together_ai/rerank/test_handler.py @@ -1,5 +1,6 @@ import json -from collections.abc import Iterator +from collections.abc import Iterator, Mapping +from typing import Final import httpx import pytest @@ -8,6 +9,8 @@ from pydantic import ValidationError import litellm from litellm.llms.together_ai.rerank.handler import TogetherAIRerank +from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfig +from litellm.types.rerank import RerankBilledUnits API_BASE = "https://api.together.example/v1" RERANKED = { @@ -94,3 +97,37 @@ def test_rerank_without_results_is_a_value_error(rerank_route: respx.Route): with pytest.raises(ValueError, match="No results found"): _rerank(is_async=False) + + +@pytest.mark.parametrize( + ("usage_fields", "expected_billed_units"), + ( + pytest.param({}, {}, id="missing-usage"), + pytest.param({"usage": {}}, {}, id="empty-usage"), + pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"), + pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"), + pytest.param( + { + "usage": { + "search_units": 2, + "total_tokens": 7, + "input_tokens": 5, + "output_tokens": 2, + "cost": 0.001, + "extra": 9, + } + }, + {"search_units": 2, "total_tokens": 7, "cost": 0.001}, + id="charged-with-token-counts", + ), + ), +) +def test_together_ai_rerank_preserves_billed_units( + usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits +) -> None: + response: Final = TogetherAIRerankConfig().transform_response( + {"results": [{"index": 0, "relevance_score": 0.9}], **usage_fields} + ) + + assert response.meta is not None + assert response.meta["billed_units"] == expected_billed_units diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 8fa731b804a..73b4224bfef 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -33,7 +33,7 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.llms.xai import RealtimeMeteredUsage -from litellm.types.rerank import RerankResponse +from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject from litellm.utils import supports_prompt_caching @@ -1615,6 +1615,45 @@ def test_rerank_completion_cost_uses_the_provider_reported_cost(): assert cost == pytest.approx(0.001) +@pytest.mark.parametrize("call_type", ("rerank", "arerank")) +@pytest.mark.parametrize( + ("billed_units", "expected_reported_cost"), + ( + pytest.param({"search_units": 2, "cost": 0.001}, 0.001, id="provider-overrides-map"), + pytest.param({"search_units": 2, "cost": 0.0}, 0.0, id="explicit-zero-overrides-map"), + pytest.param({"search_units": 2}, None, id="absent-cost-uses-map"), + pytest.param({"search_units": 2, "cost": None}, None, id="null-cost-uses-map"), + ), +) +def test_rerank_reported_cost_precedence_and_fallback( + call_type: Literal["rerank", "arerank"], + billed_units: RerankBilledUnits, + expected_reported_cost: float | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + model: Final = "cohere/rerank-reported-cost-test" + price_per_query: Final = 0.002 + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_query": price_per_query, "litellm_provider": "cohere", "mode": "rerank"}, + ) + response: Final = RerankResponse( + results=[{"index": 0, "relevance_score": 0.9}], + meta={"billed_units": billed_units}, + ) + + cost: Final = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="cohere", + call_type=call_type, + ) + + expected_cost: Final = 2 * price_per_query if expected_reported_cost is None else expected_reported_cost + assert cost == pytest.approx(expected_cost) + + def test_rerank_custom_pricing_wins_over_the_provider_reported_cost(): from litellm import Router