fix(rerank): validate billed units and expand cost regressions

This commit is contained in:
Tannmay-ts 2026-10-22 00:48:20 +05:30
parent ac1cda4af9
commit 6328faf044
6 changed files with 197 additions and 4 deletions

View file

@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works
from typing import Final
from pydantic import TypeAdapter
from litellm._uuid import uuid
from litellm.types.llms.bedrock import (
BedrockRerankBedrockRerankingConfiguration,
@ -27,6 +29,8 @@ from litellm.types.rerank import (
RerankTokens,
)
_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits)
class BedrockRerankConfig:
def _transform_sources(self, documents: list[str | dict]) -> list[BedrockRerankSource]:
@ -90,7 +94,7 @@ class BedrockRerankConfig:
example input:
{"results":[{"index":0,"relevanceScore":0.6847912669181824},{"index":1,"relevanceScore":0.5980774760246277}]}
"""
_billed_units = RerankBilledUnits(**response.get("usage", {"search_units": 1})) # by default 1 search unit
_billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {"search_units": 1}))
_tokens: Final = RerankTokens(**response.get("usage", {}))
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)

View file

@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works
from typing import Final
from pydantic import TypeAdapter
from litellm._uuid import uuid
from litellm.types.rerank import (
RerankBilledUnits,
@ -16,10 +18,12 @@ from litellm.types.rerank import (
RerankTokens,
)
_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits)
class TogetherAIRerankConfig:
def _transform_response(self, response: dict) -> RerankResponse:
_billed_units: Final = RerankBilledUnits(**response.get("usage", {}))
_billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {}))
_tokens: Final = RerankTokens(**response.get("usage", {}))
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)

View file

@ -6,6 +6,8 @@ forward_client_headers_to_llm_api were not being passed to Bedrock rerank provid
"""
import json
from collections.abc import Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
@ -15,7 +17,9 @@ from pydantic import ValidationError
import litellm
from litellm.llms.bedrock.base_aws_llm import Boto3CredentialsInfo
from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
from litellm.llms.bedrock.rerank.transformation import BedrockRerankConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.rerank import RerankBilledUnits
@pytest.fixture(autouse=True)
@ -565,3 +569,37 @@ async def test_bedrock_arerank_rejects_non_object_bodies(payload: object):
await _rerank_through_transport(payload, is_async=True)
assert "sensitive-document" not in str(exc_info.value)
@pytest.mark.parametrize(
("usage_fields", "expected_billed_units"),
(
pytest.param({}, {"search_units": 1}, id="missing-usage"),
pytest.param({"usage": {}}, {}, id="empty-usage"),
pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"),
pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"),
pytest.param(
{
"usage": {
"search_units": 2,
"total_tokens": 7,
"input_tokens": 5,
"output_tokens": 2,
"cost": 0.001,
"extra": 9,
}
},
{"search_units": 2, "total_tokens": 7, "cost": 0.001},
id="charged-with-token-counts",
),
),
)
def test_bedrock_rerank_preserves_billed_units(
usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits
) -> None:
response: Final = BedrockRerankConfig().transform_response(
{"results": [{"index": 0, "relevanceScore": 0.9}], **usage_fields}
)
assert response.meta is not None
assert response.meta["billed_units"] == expected_billed_units

View file

@ -1,4 +1,5 @@
import json
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
@ -7,6 +8,7 @@ import respx
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.rerank import RerankBilledUnits, RerankResponse
SCALEWAY_RERANK_BODY = {
"id": "rerank-a89e6d7b8b97492ea81569c65fbfff49",
@ -134,3 +136,72 @@ async def test_scaleway_arerank_posts_to_the_documented_endpoint():
assert client.post.await_args.kwargs["url"] == "https://api.scaleway.ai/v1/rerank"
assert client.post.await_args.kwargs["headers"]["authorization"] == "Bearer scw-key"
assert [r["index"] for r in response.results] == [1, 0]
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", (True, False), ids=("sync", "async"))
@pytest.mark.parametrize(
("billed_units", "reported_cost"),
(
pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.001}, 0.001, id="charged"),
pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.0}, 0.0, id="free"),
pytest.param({"search_units": 2, "total_tokens": 2}, None, id="absent-cost"),
pytest.param({"search_units": 2, "total_tokens": 2, "cost": None}, None, id="null-cost"),
),
)
async def test_scaleway_rerank_preserves_openrouter_reported_cost(
sync_mode: bool,
billed_units: RerankBilledUnits,
reported_cost: float | None,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setattr(litellm, "cache", None)
price_per_token: Final = 0.002
monkeypatch.setitem(
litellm.model_cost,
"scaleway/cohere/rerank-v3.5",
{"input_cost_per_token": price_per_token, "litellm_provider": "scaleway", "mode": "rerank"},
)
route: Final = respx_mock.post("https://openrouter.ai/api/v1/rerank")
route.return_value = httpx.Response(
200,
json={
"id": "openrouter-rerank-cost-test",
"results": [{"index": 0, "relevance_score": 0.9}],
"usage": billed_units,
},
)
response: Final = (
litellm.rerank(
model="scaleway/cohere/rerank-v3.5",
api_base="https://openrouter.ai/api/v1",
api_key="test-openrouter-key",
query="the customer disputed the deployment terms",
documents=["the customer disputed the deployment terms"],
top_n=1,
)
if sync_mode
else await litellm.arerank(
model="scaleway/cohere/rerank-v3.5",
api_base="https://openrouter.ai/api/v1",
api_key="test-openrouter-key",
query="the customer disputed the deployment terms",
documents=["the customer disputed the deployment terms"],
top_n=1,
)
)
assert isinstance(response, RerankResponse)
assert route.call_count == 1
assert json.loads(route.calls[0].request.content) == {
"model": "cohere/rerank-v3.5",
"query": "the customer disputed the deployment terms",
"documents": ["the customer disputed the deployment terms"],
"top_n": 1,
}
assert response.meta == {"billed_units": billed_units, "tokens": {}}
expected_cost: Final = 2 * price_per_token if reported_cost is None else reported_cost
assert response._hidden_params["response_cost"] == pytest.approx(expected_cost)

View file

@ -1,5 +1,6 @@
import json
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from typing import Final
import httpx
import pytest
@ -8,6 +9,8 @@ from pydantic import ValidationError
import litellm
from litellm.llms.together_ai.rerank.handler import TogetherAIRerank
from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfig
from litellm.types.rerank import RerankBilledUnits
API_BASE = "https://api.together.example/v1"
RERANKED = {
@ -94,3 +97,37 @@ def test_rerank_without_results_is_a_value_error(rerank_route: respx.Route):
with pytest.raises(ValueError, match="No results found"):
_rerank(is_async=False)
@pytest.mark.parametrize(
("usage_fields", "expected_billed_units"),
(
pytest.param({}, {}, id="missing-usage"),
pytest.param({"usage": {}}, {}, id="empty-usage"),
pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"),
pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"),
pytest.param(
{
"usage": {
"search_units": 2,
"total_tokens": 7,
"input_tokens": 5,
"output_tokens": 2,
"cost": 0.001,
"extra": 9,
}
},
{"search_units": 2, "total_tokens": 7, "cost": 0.001},
id="charged-with-token-counts",
),
),
)
def test_together_ai_rerank_preserves_billed_units(
usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits
) -> None:
response: Final = TogetherAIRerankConfig().transform_response(
{"results": [{"index": 0, "relevance_score": 0.9}], **usage_fields}
)
assert response.meta is not None
assert response.meta["billed_units"] == expected_billed_units

View file

@ -33,7 +33,7 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.llms.xai import RealtimeMeteredUsage
from litellm.types.rerank import RerankResponse
from litellm.types.rerank import RerankBilledUnits, RerankResponse
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
from litellm.types.videos.main import VideoObject
from litellm.utils import supports_prompt_caching
@ -1615,6 +1615,45 @@ def test_rerank_completion_cost_uses_the_provider_reported_cost():
assert cost == pytest.approx(0.001)
@pytest.mark.parametrize("call_type", ("rerank", "arerank"))
@pytest.mark.parametrize(
("billed_units", "expected_reported_cost"),
(
pytest.param({"search_units": 2, "cost": 0.001}, 0.001, id="provider-overrides-map"),
pytest.param({"search_units": 2, "cost": 0.0}, 0.0, id="explicit-zero-overrides-map"),
pytest.param({"search_units": 2}, None, id="absent-cost-uses-map"),
pytest.param({"search_units": 2, "cost": None}, None, id="null-cost-uses-map"),
),
)
def test_rerank_reported_cost_precedence_and_fallback(
call_type: Literal["rerank", "arerank"],
billed_units: RerankBilledUnits,
expected_reported_cost: float | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
model: Final = "cohere/rerank-reported-cost-test"
price_per_query: Final = 0.002
monkeypatch.setitem(
litellm.model_cost,
model,
{"input_cost_per_query": price_per_query, "litellm_provider": "cohere", "mode": "rerank"},
)
response: Final = RerankResponse(
results=[{"index": 0, "relevance_score": 0.9}],
meta={"billed_units": billed_units},
)
cost: Final = completion_cost(
completion_response=response,
model=model,
custom_llm_provider="cohere",
call_type=call_type,
)
expected_cost: Final = 2 * price_per_query if expected_reported_cost is None else expected_reported_cost
assert cost == pytest.approx(expected_cost)
def test_rerank_custom_pricing_wins_over_the_provider_reported_cost():
from litellm import Router