mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(rerank): keep shared budget reservation unchanged
Restore shared proxy reservation and token counting to main, remove the associated reservation tests, and restore the original rerank call-count helper Keep rerank response cost aggregation and explicit duration pricing Validation passed with 748 focused tests and make check
This commit is contained in:
parent
4343c562ff
commit
55692c25dc
5 changed files with 28 additions and 293 deletions
|
|
@ -39,18 +39,9 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
tag_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
count_input_tokens,
|
||||
count_input_tokens_for_model,
|
||||
count_text_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
|
||||
DecisionRerankRequest,
|
||||
LiteLLMDecisionsRerankConfig,
|
||||
)
|
||||
from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
|
|
@ -83,7 +74,6 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
|
|||
|
||||
_CACHED_VALUE: Final = TypeAdapter(object)
|
||||
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_REQUEST_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _CounterReservationUnavailable(Exception):
|
||||
|
|
@ -1351,34 +1341,26 @@ def _estimate_request_input_cost_for_model(
|
|||
llm_router: Router | None,
|
||||
input_tokens: int | None = None,
|
||||
) -> float | None:
|
||||
body: Final = _REQUEST_BODY.validate_python(request_body)
|
||||
estimates: Final = [
|
||||
_input_cost_for_cost_info(
|
||||
request_body=body,
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(
|
||||
model=model,
|
||||
llm_router=llm_router,
|
||||
custom_llm_provider=_request_string(body, "custom_llm_provider"),
|
||||
api_base=_request_string(body, "api_base"),
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
]
|
||||
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
|
||||
return max(valid_estimates) if valid_estimates else None
|
||||
|
||||
|
||||
def _input_cost_for_cost_info(
|
||||
request_body: dict[str, object],
|
||||
request_body: dict,
|
||||
route: str,
|
||||
model: str,
|
||||
model_info: Mapping[str, object],
|
||||
input_tokens: int | None = None,
|
||||
llm_router: Router | None = None,
|
||||
) -> float | None:
|
||||
estimated_input_tokens: Final = _estimate_input_tokens(
|
||||
request_body=request_body,
|
||||
|
|
@ -1386,7 +1368,6 @@ def _input_cost_for_cost_info(
|
|||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if estimated_input_tokens is None:
|
||||
return None
|
||||
|
|
@ -1408,34 +1389,26 @@ def _estimate_request_max_cost_for_model(
|
|||
llm_router: Router | None,
|
||||
input_tokens: int | None = None,
|
||||
) -> float | None:
|
||||
body: Final = _REQUEST_BODY.validate_python(request_body)
|
||||
estimates: Final = [
|
||||
_max_cost_for_cost_info(
|
||||
request_body=body,
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(
|
||||
model=model,
|
||||
llm_router=llm_router,
|
||||
custom_llm_provider=_request_string(body, "custom_llm_provider"),
|
||||
api_base=_request_string(body, "api_base"),
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
]
|
||||
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
|
||||
return max(valid_estimates) if valid_estimates else None
|
||||
|
||||
|
||||
def _max_cost_for_cost_info(
|
||||
request_body: dict[str, object],
|
||||
request_body: dict,
|
||||
route: str,
|
||||
model: str,
|
||||
model_info: Mapping[str, object],
|
||||
input_tokens: int | None = None,
|
||||
llm_router: Router | None = None,
|
||||
) -> float | None:
|
||||
image_cost: Final = _estimate_image_generation_cost(
|
||||
request_body=request_body,
|
||||
|
|
@ -1450,13 +1423,11 @@ def _max_cost_for_cost_info(
|
|||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
output_tokens: Final = _estimate_output_tokens(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model_info=model_info,
|
||||
decision_requests=_decision_rerank_requests(request_body, route, model, llm_router),
|
||||
)
|
||||
if estimated_input_tokens is None or output_tokens is None:
|
||||
return None
|
||||
|
|
@ -1536,26 +1507,17 @@ def _estimate_image_generation_cost(
|
|||
def _get_model_cost_info(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> Mapping[str, object] | None:
|
||||
if llm_router is not None:
|
||||
model_group_info: Final = llm_router.cached_model_group_info(model)
|
||||
if model_group_info is not None:
|
||||
return model_group_info.model_dump()
|
||||
resolved_model, provider, _, _ = (
|
||||
litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base)
|
||||
if api_base is not None
|
||||
else (model, custom_llm_provider, None, None)
|
||||
)
|
||||
return dict(litellm.get_model_info(model=resolved_model, custom_llm_provider=provider, api_base=api_base))
|
||||
return dict(litellm.get_model_info(model=model))
|
||||
|
||||
|
||||
def _get_model_cost_infos(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""Cost-info candidates to estimate a request against for one model group.
|
||||
|
||||
|
|
@ -1566,9 +1528,7 @@ def _get_model_cost_infos(
|
|||
deployment can never leave the request under-reserved.
|
||||
"""
|
||||
try:
|
||||
base: Final = _get_model_cost_info(
|
||||
model=model, llm_router=llm_router, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
)
|
||||
base: Final = _get_model_cost_info(model=model, llm_router=llm_router)
|
||||
if base is None:
|
||||
return []
|
||||
tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router)
|
||||
|
|
@ -1634,109 +1594,24 @@ async def count_request_input_tokens(
|
|||
"""Input-token count per candidate model, counted once per request.
|
||||
|
||||
The counts are reused by both the max-cost and the input-cost estimate."""
|
||||
body: Final = _REQUEST_BODY.validate_python(request_body)
|
||||
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
|
||||
if not models:
|
||||
return MappingProxyType({})
|
||||
decision_requests: Final = {
|
||||
model: requests for model in models if (requests := _decision_rerank_requests(body, route, model, llm_router))
|
||||
}
|
||||
ordinary_counts: Final = await count_input_tokens(
|
||||
request_body=request_body,
|
||||
raw_body=raw_body,
|
||||
models=models,
|
||||
)
|
||||
decision_counts: Final = await asyncio.gather(
|
||||
*(asyncio.to_thread(_decision_rerank_input_tokens, requests) for requests in decision_requests.values())
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**ordinary_counts,
|
||||
**{
|
||||
model: max(tokens, ordinary_counts.get(model, 0))
|
||||
for model, tokens in zip(decision_requests, decision_counts, strict=True)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _request_string(request_body: Mapping[str, object], field: str) -> str | None:
|
||||
value: Final = request_body.get(field)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _decision_rerank_request(
|
||||
request_body: Mapping[str, object], model: str, custom_llm_provider: str | None, api_base: str | None
|
||||
) -> tuple[str, DecisionRerankRequest] | None:
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
|
||||
)
|
||||
except litellm.BadRequestError:
|
||||
return None
|
||||
strategy: Final = get_decision_rerank_strategy(resolved_model, provider)
|
||||
if strategy is None:
|
||||
return None
|
||||
return (
|
||||
f"{provider}/{resolved_model}",
|
||||
LiteLLMDecisionsRerankConfig.validate_request(request_body, model=model, provider=provider, strategy=strategy),
|
||||
)
|
||||
|
||||
|
||||
def _decision_rerank_requests(
|
||||
request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None
|
||||
) -> tuple[tuple[str, DecisionRerankRequest], ...]:
|
||||
if "rerank" not in route:
|
||||
return ()
|
||||
deployments: Final = llm_router.get_model_list(model_name=model) if llm_router is not None else ()
|
||||
models: Final = tuple(
|
||||
dict.fromkeys(
|
||||
(
|
||||
deployment["litellm_params"]["model"],
|
||||
_request_string(deployment["litellm_params"], "custom_llm_provider"),
|
||||
_request_string(deployment["litellm_params"], "api_base"),
|
||||
)
|
||||
for deployment in deployments or ()
|
||||
if "model" in deployment["litellm_params"]
|
||||
)
|
||||
)
|
||||
return tuple(
|
||||
request
|
||||
for backend_model, provider, api_base in models
|
||||
or ((model, _request_string(request_body, "custom_llm_provider"), _request_string(request_body, "api_base")),)
|
||||
if (request := _decision_rerank_request(request_body, backend_model, provider, api_base)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _decision_rerank_input_tokens(requests: Sequence[tuple[str, DecisionRerankRequest]]) -> int:
|
||||
return max(
|
||||
sum(
|
||||
count_text_tokens(
|
||||
model=model,
|
||||
text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json(
|
||||
exclude_none=True, exclude_defaults=True
|
||||
),
|
||||
)
|
||||
for index in range(request.call_count)
|
||||
)
|
||||
for model, request in requests
|
||||
)
|
||||
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
|
||||
|
||||
|
||||
def _estimate_input_tokens(
|
||||
request_body: dict[str, object],
|
||||
request_body: dict,
|
||||
route: str,
|
||||
model: str,
|
||||
model_info: Mapping[str, object],
|
||||
input_tokens: int | None = None,
|
||||
llm_router: Router | None = None,
|
||||
) -> int | None:
|
||||
if input_tokens is not None:
|
||||
return input_tokens
|
||||
decision_requests: Final = _decision_rerank_requests(request_body, route, model, llm_router)
|
||||
if decision_requests:
|
||||
return _decision_rerank_input_tokens(decision_requests)
|
||||
counted: Final = count_input_tokens_for_model(request_body=request_body, model=model)
|
||||
counted: Final = (
|
||||
input_tokens
|
||||
if input_tokens is not None
|
||||
else count_input_tokens_for_model(request_body=request_body, model=model)
|
||||
)
|
||||
if counted is not None:
|
||||
return counted
|
||||
|
||||
|
|
@ -1754,11 +1629,7 @@ def _estimate_output_tokens(
|
|||
request_body: dict,
|
||||
route: str,
|
||||
model_info: Mapping[str, object],
|
||||
decision_requests: Sequence[tuple[str, DecisionRerankRequest]] = (),
|
||||
) -> int | None:
|
||||
if decision_requests:
|
||||
per_call: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK
|
||||
return per_call * max(request.call_count for _, request in decision_requests)
|
||||
if _is_input_only_route(route=route):
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
|||
tool_choice=request_body.get("tool_choice"),
|
||||
)
|
||||
except ValueError:
|
||||
return count_text_tokens(model=model, text=request_body.get("messages"))
|
||||
return _count_text_tokens(model=model, text=request_body.get("messages"))
|
||||
if "prompt" in request_body:
|
||||
return count_text_tokens(model=model, text=request_body.get("prompt"))
|
||||
return _count_text_tokens(model=model, text=request_body.get("prompt"))
|
||||
if "input" in request_body:
|
||||
return count_text_tokens(
|
||||
return _count_text_tokens(
|
||||
model=model, text=billable_input_since_newest_compaction(request_body.get("input"))
|
||||
)
|
||||
if "query" in request_body or "documents" in request_body:
|
||||
query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query"))
|
||||
document_tokens: Final = count_text_tokens(
|
||||
query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query"))
|
||||
document_tokens: Final = _count_text_tokens(
|
||||
model=model,
|
||||
text=request_body.get("documents"),
|
||||
)
|
||||
|
|
@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def count_text_tokens(model: str, text: object) -> int:
|
||||
def _count_text_tokens(model: str, text: object) -> int:
|
||||
if text is None:
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -149,6 +149,10 @@ def _finish(
|
|||
return response
|
||||
|
||||
|
||||
def _call_count(request: DecisionRerankRequest) -> int:
|
||||
return 1 if request.strategy == "multiple_choice" else len(request.documents)
|
||||
|
||||
|
||||
def _rerank(
|
||||
prepare: Callable[[int], _DocumentCall],
|
||||
request: DecisionRerankRequest,
|
||||
|
|
@ -157,7 +161,7 @@ def _rerank(
|
|||
provider: str,
|
||||
deadline: float,
|
||||
) -> DecisionRerankResponse:
|
||||
count: Final = request.call_count
|
||||
count: Final = _call_count(request)
|
||||
workers: Final = min(count, _MAX_CONCURRENCY)
|
||||
completed: Final[SimpleQueue[DecisionRerankResult]] = SimpleQueue()
|
||||
|
||||
|
|
@ -193,7 +197,7 @@ async def _arerank(
|
|||
provider: str,
|
||||
deadline: float,
|
||||
) -> DecisionRerankResponse:
|
||||
count: Final = request.call_count
|
||||
count: Final = _call_count(request)
|
||||
workers: Final = min(count, _MAX_CONCURRENCY)
|
||||
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
|
||||
cancellation_requested: Final = asyncio.Event()
|
||||
|
|
|
|||
|
|
@ -66,10 +66,6 @@ class DecisionRerankRequest(BaseModel):
|
|||
document: Final = self.documents[index]
|
||||
return document if isinstance(document, str) else document.text
|
||||
|
||||
@property
|
||||
def call_count(self) -> int:
|
||||
return 1 if self.strategy == "multiple_choice" else len(self.documents)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DecisionRerankResult:
|
||||
|
|
|
|||
|
|
@ -25,13 +25,11 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
_get_team_member_budget_counter,
|
||||
count_request_input_tokens,
|
||||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
release_unbound_budget_reservation,
|
||||
reserve_budget_for_request,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
|
@ -222,140 +220,6 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
|
|||
assert math.isclose(after, before * 1000)
|
||||
|
||||
|
||||
def _rerank_router(backend_model: str, custom_llm_provider: str | None = None, api_base: str | None = None) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
Deployment(
|
||||
model_name="budgeted-rerank",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model=backend_model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key="sk-fake"
|
||||
),
|
||||
model_info=ModelInfo(
|
||||
id="budgeted-rerank-deployment",
|
||||
input_cost_per_token=0.001,
|
||||
output_cost_per_token=0.002,
|
||||
max_output_tokens=10,
|
||||
),
|
||||
).model_dump()
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
RERANK_QUERY: Final = "relevant query " * 100
|
||||
RERANK_BODY: Final = {
|
||||
"model": "budgeted-rerank",
|
||||
"query": RERANK_QUERY,
|
||||
"documents": ["first passage", {"text": "second passage"}, "third passage", "fourth passage"],
|
||||
"max_tokens": 1,
|
||||
}
|
||||
|
||||
|
||||
async def _reserve_rerank(router: Router, max_budget: float) -> dict[str, object] | None:
|
||||
return await reserve_budget_for_request(
|
||||
request_body=RERANK_BODY,
|
||||
route="/v1/rerank",
|
||||
llm_router=router,
|
||||
valid_token=UserAPIKeyAuth(token="decision-rerank-budget", max_budget=max_budget, spend=0.0),
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=UserApiKeyCache(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
fail_closed_budget_enforcement=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("backend_model", "call_count"),
|
||||
[("typesafe/jev-latest", 4), ("perplexity/pplx-decider-latest", 1)],
|
||||
)
|
||||
async def test_decision_rerank_reserves_transformed_inputs_and_output_for_every_call(
|
||||
spend_counter_cache: DualCache, backend_model: str, call_count: int
|
||||
) -> None:
|
||||
reservation: Final = await _reserve_rerank(_rerank_router(backend_model), max_budget=100.0)
|
||||
query_tokens: Final = litellm.token_counter(model=backend_model, text=RERANK_QUERY)
|
||||
|
||||
assert reservation is not None
|
||||
input_tokens: Final = reservation["input_tokens"]
|
||||
assert isinstance(input_tokens, int)
|
||||
assert input_tokens > query_tokens * call_count
|
||||
assert reservation["input_cost"] == pytest.approx(input_tokens * 0.001)
|
||||
assert reservation["reserved_cost"] == pytest.approx(input_tokens * 0.001 + call_count * 10 * 0.002)
|
||||
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx(
|
||||
reservation["reserved_cost"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("backend_model", ["typesafe/jev-latest", "perplexity/pplx-decider-latest"])
|
||||
async def test_decision_rerank_rejects_a_budget_that_only_covers_the_untransformed_request(
|
||||
spend_counter_cache: DualCache, backend_model: str
|
||||
) -> None:
|
||||
raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank")
|
||||
assert raw_tokens is not None
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="Estimated request cost"):
|
||||
await _reserve_rerank(_rerank_router(backend_model), max_budget=raw_tokens * 0.001 + 0.001)
|
||||
|
||||
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx(0)
|
||||
|
||||
|
||||
def test_native_rerank_remains_input_only() -> None:
|
||||
router: Final = _rerank_router("cohere/rerank-v3.5")
|
||||
raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank")
|
||||
assert raw_tokens is not None
|
||||
|
||||
estimate: Final = estimate_request_max_cost(request_body=RERANK_BODY, route="/v1/rerank", llm_router=router)
|
||||
|
||||
assert estimate == pytest.approx(raw_tokens * 0.001)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_fields", [{"documents": []}, {"top_n": 0}])
|
||||
def test_decision_rerank_admission_preserves_client_validation_errors(invalid_fields: dict[str, object]) -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as error:
|
||||
estimate_request_max_cost(
|
||||
request_body={**RERANK_BODY, **invalid_fields},
|
||||
route="/v1/rerank",
|
||||
llm_router=_rerank_router("typesafe/jev-latest"),
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider", "api_base"),
|
||||
[
|
||||
("jev-latest", "typesafe", None),
|
||||
("pplx-decider-latest", "perplexity", None),
|
||||
("pplx-decider-latest", "perplexity", "https://api.perplexity.ai"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_router", [False, True])
|
||||
async def test_decision_rerank_provider_configuration_reserves_the_same_cost_as_provider_prefix(
|
||||
model: str, provider: str, api_base: str | None, use_router: bool
|
||||
) -> None:
|
||||
prefixed_model: Final = f"{provider}/{model}"
|
||||
prefixed_router: Final = _rerank_router(prefixed_model) if use_router else None
|
||||
explicit_provider: Final = None if api_base is not None else provider
|
||||
explicit_router: Final = _rerank_router(model, explicit_provider, api_base) if use_router else None
|
||||
prefixed_body: Final = RERANK_BODY if use_router else {**RERANK_BODY, "model": prefixed_model}
|
||||
explicit_body: Final = (
|
||||
RERANK_BODY
|
||||
if use_router
|
||||
else {**RERANK_BODY, "model": model, "custom_llm_provider": explicit_provider, "api_base": api_base}
|
||||
)
|
||||
prefixed_counts: Final = await count_request_input_tokens(prefixed_body, "/v1/rerank", prefixed_router)
|
||||
explicit_counts: Final = await count_request_input_tokens(explicit_body, "/v1/rerank", explicit_router)
|
||||
prefixed_cost: Final = estimate_request_max_cost(prefixed_body, "/v1/rerank", prefixed_router, prefixed_counts)
|
||||
explicit_cost: Final = estimate_request_max_cost(explicit_body, "/v1/rerank", explicit_router, explicit_counts)
|
||||
|
||||
assert prefixed_cost is not None and prefixed_cost > 0
|
||||
assert explicit_cost == pytest.approx(prefixed_cost)
|
||||
assert tuple(explicit_counts.values()) == tuple(prefixed_counts.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"expiry_offset, expected_max_budget",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue