mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(rerank): validate billed units and expand cost regressions
This commit is contained in:
parent
ac1cda4af9
commit
6328faf044
6 changed files with 197 additions and 4 deletions
|
|
@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works
|
|||
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.types.llms.bedrock import (
|
||||
BedrockRerankBedrockRerankingConfiguration,
|
||||
|
|
@ -27,6 +29,8 @@ from litellm.types.rerank import (
|
|||
RerankTokens,
|
||||
)
|
||||
|
||||
_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits)
|
||||
|
||||
|
||||
class BedrockRerankConfig:
|
||||
def _transform_sources(self, documents: list[str | dict]) -> list[BedrockRerankSource]:
|
||||
|
|
@ -90,7 +94,7 @@ class BedrockRerankConfig:
|
|||
example input:
|
||||
{"results":[{"index":0,"relevanceScore":0.6847912669181824},{"index":1,"relevanceScore":0.5980774760246277}]}
|
||||
"""
|
||||
_billed_units = RerankBilledUnits(**response.get("usage", {"search_units": 1})) # by default 1 search unit
|
||||
_billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {"search_units": 1}))
|
||||
_tokens: Final = RerankTokens(**response.get("usage", {}))
|
||||
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works
|
|||
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.types.rerank import (
|
||||
RerankBilledUnits,
|
||||
|
|
@ -16,10 +18,12 @@ from litellm.types.rerank import (
|
|||
RerankTokens,
|
||||
)
|
||||
|
||||
_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits)
|
||||
|
||||
|
||||
class TogetherAIRerankConfig:
|
||||
def _transform_response(self, response: dict) -> RerankResponse:
|
||||
_billed_units: Final = RerankBilledUnits(**response.get("usage", {}))
|
||||
_billed_units: Final = _BILLED_UNITS.validate_python(response.get("usage", {}))
|
||||
_tokens: Final = RerankTokens(**response.get("usage", {}))
|
||||
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ forward_client_headers_to_llm_api were not being passed to Bedrock rerank provid
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -15,7 +17,9 @@ from pydantic import ValidationError
|
|||
import litellm
|
||||
from litellm.llms.bedrock.base_aws_llm import Boto3CredentialsInfo
|
||||
from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
|
||||
from litellm.llms.bedrock.rerank.transformation import BedrockRerankConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.rerank import RerankBilledUnits
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -565,3 +569,37 @@ async def test_bedrock_arerank_rejects_non_object_bodies(payload: object):
|
|||
await _rerank_through_transport(payload, is_async=True)
|
||||
|
||||
assert "sensitive-document" not in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("usage_fields", "expected_billed_units"),
|
||||
(
|
||||
pytest.param({}, {"search_units": 1}, id="missing-usage"),
|
||||
pytest.param({"usage": {}}, {}, id="empty-usage"),
|
||||
pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"),
|
||||
pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"),
|
||||
pytest.param(
|
||||
{
|
||||
"usage": {
|
||||
"search_units": 2,
|
||||
"total_tokens": 7,
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 2,
|
||||
"cost": 0.001,
|
||||
"extra": 9,
|
||||
}
|
||||
},
|
||||
{"search_units": 2, "total_tokens": 7, "cost": 0.001},
|
||||
id="charged-with-token-counts",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_bedrock_rerank_preserves_billed_units(
|
||||
usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits
|
||||
) -> None:
|
||||
response: Final = BedrockRerankConfig().transform_response(
|
||||
{"results": [{"index": 0, "relevanceScore": 0.9}], **usage_fields}
|
||||
)
|
||||
|
||||
assert response.meta is not None
|
||||
assert response.meta["billed_units"] == expected_billed_units
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -7,6 +8,7 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.types.rerank import RerankBilledUnits, RerankResponse
|
||||
|
||||
SCALEWAY_RERANK_BODY = {
|
||||
"id": "rerank-a89e6d7b8b97492ea81569c65fbfff49",
|
||||
|
|
@ -134,3 +136,72 @@ async def test_scaleway_arerank_posts_to_the_documented_endpoint():
|
|||
assert client.post.await_args.kwargs["url"] == "https://api.scaleway.ai/v1/rerank"
|
||||
assert client.post.await_args.kwargs["headers"]["authorization"] == "Bearer scw-key"
|
||||
assert [r["index"] for r in response.results] == [1, 0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", (True, False), ids=("sync", "async"))
|
||||
@pytest.mark.parametrize(
|
||||
("billed_units", "reported_cost"),
|
||||
(
|
||||
pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.001}, 0.001, id="charged"),
|
||||
pytest.param({"search_units": 2, "total_tokens": 2, "cost": 0.0}, 0.0, id="free"),
|
||||
pytest.param({"search_units": 2, "total_tokens": 2}, None, id="absent-cost"),
|
||||
pytest.param({"search_units": 2, "total_tokens": 2, "cost": None}, None, id="null-cost"),
|
||||
),
|
||||
)
|
||||
async def test_scaleway_rerank_preserves_openrouter_reported_cost(
|
||||
sync_mode: bool,
|
||||
billed_units: RerankBilledUnits,
|
||||
reported_cost: float | None,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setattr(litellm, "cache", None)
|
||||
price_per_token: Final = 0.002
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"scaleway/cohere/rerank-v3.5",
|
||||
{"input_cost_per_token": price_per_token, "litellm_provider": "scaleway", "mode": "rerank"},
|
||||
)
|
||||
route: Final = respx_mock.post("https://openrouter.ai/api/v1/rerank")
|
||||
route.return_value = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "openrouter-rerank-cost-test",
|
||||
"results": [{"index": 0, "relevance_score": 0.9}],
|
||||
"usage": billed_units,
|
||||
},
|
||||
)
|
||||
|
||||
response: Final = (
|
||||
litellm.rerank(
|
||||
model="scaleway/cohere/rerank-v3.5",
|
||||
api_base="https://openrouter.ai/api/v1",
|
||||
api_key="test-openrouter-key",
|
||||
query="the customer disputed the deployment terms",
|
||||
documents=["the customer disputed the deployment terms"],
|
||||
top_n=1,
|
||||
)
|
||||
if sync_mode
|
||||
else await litellm.arerank(
|
||||
model="scaleway/cohere/rerank-v3.5",
|
||||
api_base="https://openrouter.ai/api/v1",
|
||||
api_key="test-openrouter-key",
|
||||
query="the customer disputed the deployment terms",
|
||||
documents=["the customer disputed the deployment terms"],
|
||||
top_n=1,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(response, RerankResponse)
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "cohere/rerank-v3.5",
|
||||
"query": "the customer disputed the deployment terms",
|
||||
"documents": ["the customer disputed the deployment terms"],
|
||||
"top_n": 1,
|
||||
}
|
||||
assert response.meta == {"billed_units": billed_units, "tokens": {}}
|
||||
expected_cost: Final = 2 * price_per_token if reported_cost is None else reported_cost
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(expected_cost)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterator, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -8,6 +9,8 @@ from pydantic import ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.llms.together_ai.rerank.handler import TogetherAIRerank
|
||||
from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfig
|
||||
from litellm.types.rerank import RerankBilledUnits
|
||||
|
||||
API_BASE = "https://api.together.example/v1"
|
||||
RERANKED = {
|
||||
|
|
@ -94,3 +97,37 @@ def test_rerank_without_results_is_a_value_error(rerank_route: respx.Route):
|
|||
|
||||
with pytest.raises(ValueError, match="No results found"):
|
||||
_rerank(is_async=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("usage_fields", "expected_billed_units"),
|
||||
(
|
||||
pytest.param({}, {}, id="missing-usage"),
|
||||
pytest.param({"usage": {}}, {}, id="empty-usage"),
|
||||
pytest.param({"usage": {"search_units": 1, "cost": None}}, {"search_units": 1, "cost": None}, id="null-cost"),
|
||||
pytest.param({"usage": {"search_units": 1, "cost": 0.0}}, {"search_units": 1, "cost": 0.0}, id="free"),
|
||||
pytest.param(
|
||||
{
|
||||
"usage": {
|
||||
"search_units": 2,
|
||||
"total_tokens": 7,
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 2,
|
||||
"cost": 0.001,
|
||||
"extra": 9,
|
||||
}
|
||||
},
|
||||
{"search_units": 2, "total_tokens": 7, "cost": 0.001},
|
||||
id="charged-with-token-counts",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_together_ai_rerank_preserves_billed_units(
|
||||
usage_fields: Mapping[str, object], expected_billed_units: RerankBilledUnits
|
||||
) -> None:
|
||||
response: Final = TogetherAIRerankConfig().transform_response(
|
||||
{"results": [{"index": 0, "relevance_score": 0.9}], **usage_fields}
|
||||
)
|
||||
|
||||
assert response.meta is not None
|
||||
assert response.meta["billed_units"] == expected_billed_units
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego
|
|||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.rerank import RerankBilledUnits, RerankResponse
|
||||
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
@ -1615,6 +1615,45 @@ def test_rerank_completion_cost_uses_the_provider_reported_cost():
|
|||
assert cost == pytest.approx(0.001)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_type", ("rerank", "arerank"))
|
||||
@pytest.mark.parametrize(
|
||||
("billed_units", "expected_reported_cost"),
|
||||
(
|
||||
pytest.param({"search_units": 2, "cost": 0.001}, 0.001, id="provider-overrides-map"),
|
||||
pytest.param({"search_units": 2, "cost": 0.0}, 0.0, id="explicit-zero-overrides-map"),
|
||||
pytest.param({"search_units": 2}, None, id="absent-cost-uses-map"),
|
||||
pytest.param({"search_units": 2, "cost": None}, None, id="null-cost-uses-map"),
|
||||
),
|
||||
)
|
||||
def test_rerank_reported_cost_precedence_and_fallback(
|
||||
call_type: Literal["rerank", "arerank"],
|
||||
billed_units: RerankBilledUnits,
|
||||
expected_reported_cost: float | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model: Final = "cohere/rerank-reported-cost-test"
|
||||
price_per_query: Final = 0.002
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"input_cost_per_query": price_per_query, "litellm_provider": "cohere", "mode": "rerank"},
|
||||
)
|
||||
response: Final = RerankResponse(
|
||||
results=[{"index": 0, "relevance_score": 0.9}],
|
||||
meta={"billed_units": billed_units},
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="cohere",
|
||||
call_type=call_type,
|
||||
)
|
||||
|
||||
expected_cost: Final = 2 * price_per_query if expected_reported_cost is None else expected_reported_cost
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_rerank_custom_pricing_wins_over_the_provider_reported_cost():
|
||||
from litellm import Router
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue