fix(rerank): keep shared budget reservation unchanged

Restore shared proxy reservation and token counting to main, remove the associated reservation tests, and restore the original rerank call-count helper

Keep rerank response cost aggregation and explicit duration pricing

Validation passed with 748 focused tests and make check
This commit is contained in:
Ishaan 2026-10-23 12:11:02 -07:00
parent 4343c562ff
commit 55692c25dc
5 changed files with 28 additions and 293 deletions

View file

@ -39,18 +39,9 @@ from litellm.proxy.common_utils.user_api_key_cache import (
tag_cache_key,
team_membership_reservation_cache_key,
)
from litellm.proxy.spend_tracking.input_tokens import (
count_input_tokens,
count_input_tokens_for_model,
count_text_tokens,
)
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
DecisionRerankRequest,
LiteLLMDecisionsRerankConfig,
)
from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy
from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
@ -83,7 +74,6 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
_CACHED_VALUE: Final = TypeAdapter(object)
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_REQUEST_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
class _CounterReservationUnavailable(Exception):
@ -1351,34 +1341,26 @@ def _estimate_request_input_cost_for_model(
llm_router: Router | None,
input_tokens: int | None = None,
) -> float | None:
body: Final = _REQUEST_BODY.validate_python(request_body)
estimates: Final = [
_input_cost_for_cost_info(
request_body=body,
request_body=request_body,
route=route,
model=model,
model_info=model_info,
input_tokens=input_tokens,
llm_router=llm_router,
)
for model_info in _get_model_cost_infos(
model=model,
llm_router=llm_router,
custom_llm_provider=_request_string(body, "custom_llm_provider"),
api_base=_request_string(body, "api_base"),
)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
]
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
return max(valid_estimates) if valid_estimates else None
def _input_cost_for_cost_info(
request_body: dict[str, object],
request_body: dict,
route: str,
model: str,
model_info: Mapping[str, object],
input_tokens: int | None = None,
llm_router: Router | None = None,
) -> float | None:
estimated_input_tokens: Final = _estimate_input_tokens(
request_body=request_body,
@ -1386,7 +1368,6 @@ def _input_cost_for_cost_info(
model=model,
model_info=model_info,
input_tokens=input_tokens,
llm_router=llm_router,
)
if estimated_input_tokens is None:
return None
@ -1408,34 +1389,26 @@ def _estimate_request_max_cost_for_model(
llm_router: Router | None,
input_tokens: int | None = None,
) -> float | None:
body: Final = _REQUEST_BODY.validate_python(request_body)
estimates: Final = [
_max_cost_for_cost_info(
request_body=body,
request_body=request_body,
route=route,
model=model,
model_info=model_info,
input_tokens=input_tokens,
llm_router=llm_router,
)
for model_info in _get_model_cost_infos(
model=model,
llm_router=llm_router,
custom_llm_provider=_request_string(body, "custom_llm_provider"),
api_base=_request_string(body, "api_base"),
)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
]
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
return max(valid_estimates) if valid_estimates else None
def _max_cost_for_cost_info(
request_body: dict[str, object],
request_body: dict,
route: str,
model: str,
model_info: Mapping[str, object],
input_tokens: int | None = None,
llm_router: Router | None = None,
) -> float | None:
image_cost: Final = _estimate_image_generation_cost(
request_body=request_body,
@ -1450,13 +1423,11 @@ def _max_cost_for_cost_info(
model=model,
model_info=model_info,
input_tokens=input_tokens,
llm_router=llm_router,
)
output_tokens: Final = _estimate_output_tokens(
request_body=request_body,
route=route,
model_info=model_info,
decision_requests=_decision_rerank_requests(request_body, route, model, llm_router),
)
if estimated_input_tokens is None or output_tokens is None:
return None
@ -1536,26 +1507,17 @@ def _estimate_image_generation_cost(
def _get_model_cost_info(
model: str,
llm_router: Router | None,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> Mapping[str, object] | None:
if llm_router is not None:
model_group_info: Final = llm_router.cached_model_group_info(model)
if model_group_info is not None:
return model_group_info.model_dump()
resolved_model, provider, _, _ = (
litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base)
if api_base is not None
else (model, custom_llm_provider, None, None)
)
return dict(litellm.get_model_info(model=resolved_model, custom_llm_provider=provider, api_base=api_base))
return dict(litellm.get_model_info(model=model))
def _get_model_cost_infos(
model: str,
llm_router: Router | None,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> Sequence[Mapping[str, object]]:
"""Cost-info candidates to estimate a request against for one model group.
@ -1566,9 +1528,7 @@ def _get_model_cost_infos(
deployment can never leave the request under-reserved.
"""
try:
base: Final = _get_model_cost_info(
model=model, llm_router=llm_router, custom_llm_provider=custom_llm_provider, api_base=api_base
)
base: Final = _get_model_cost_info(model=model, llm_router=llm_router)
if base is None:
return []
tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router)
@ -1634,109 +1594,24 @@ async def count_request_input_tokens(
"""Input-token count per candidate model, counted once per request.
The counts are reused by both the max-cost and the input-cost estimate."""
body: Final = _REQUEST_BODY.validate_python(request_body)
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
if not models:
return MappingProxyType({})
decision_requests: Final = {
model: requests for model in models if (requests := _decision_rerank_requests(body, route, model, llm_router))
}
ordinary_counts: Final = await count_input_tokens(
request_body=request_body,
raw_body=raw_body,
models=models,
)
decision_counts: Final = await asyncio.gather(
*(asyncio.to_thread(_decision_rerank_input_tokens, requests) for requests in decision_requests.values())
)
return MappingProxyType(
{
**ordinary_counts,
**{
model: max(tokens, ordinary_counts.get(model, 0))
for model, tokens in zip(decision_requests, decision_counts, strict=True)
},
}
)
def _request_string(request_body: Mapping[str, object], field: str) -> str | None:
value: Final = request_body.get(field)
return value if isinstance(value, str) else None
def _decision_rerank_request(
request_body: Mapping[str, object], model: str, custom_llm_provider: str | None, api_base: str | None
) -> tuple[str, DecisionRerankRequest] | None:
try:
resolved_model, provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base
)
except litellm.BadRequestError:
return None
strategy: Final = get_decision_rerank_strategy(resolved_model, provider)
if strategy is None:
return None
return (
f"{provider}/{resolved_model}",
LiteLLMDecisionsRerankConfig.validate_request(request_body, model=model, provider=provider, strategy=strategy),
)
def _decision_rerank_requests(
request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None
) -> tuple[tuple[str, DecisionRerankRequest], ...]:
if "rerank" not in route:
return ()
deployments: Final = llm_router.get_model_list(model_name=model) if llm_router is not None else ()
models: Final = tuple(
dict.fromkeys(
(
deployment["litellm_params"]["model"],
_request_string(deployment["litellm_params"], "custom_llm_provider"),
_request_string(deployment["litellm_params"], "api_base"),
)
for deployment in deployments or ()
if "model" in deployment["litellm_params"]
)
)
return tuple(
request
for backend_model, provider, api_base in models
or ((model, _request_string(request_body, "custom_llm_provider"), _request_string(request_body, "api_base")),)
if (request := _decision_rerank_request(request_body, backend_model, provider, api_base)) is not None
)
def _decision_rerank_input_tokens(requests: Sequence[tuple[str, DecisionRerankRequest]]) -> int:
return max(
sum(
count_text_tokens(
model=model,
text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json(
exclude_none=True, exclude_defaults=True
),
)
for index in range(request.call_count)
)
for model, request in requests
)
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
def _estimate_input_tokens(
request_body: dict[str, object],
request_body: dict,
route: str,
model: str,
model_info: Mapping[str, object],
input_tokens: int | None = None,
llm_router: Router | None = None,
) -> int | None:
if input_tokens is not None:
return input_tokens
decision_requests: Final = _decision_rerank_requests(request_body, route, model, llm_router)
if decision_requests:
return _decision_rerank_input_tokens(decision_requests)
counted: Final = count_input_tokens_for_model(request_body=request_body, model=model)
counted: Final = (
input_tokens
if input_tokens is not None
else count_input_tokens_for_model(request_body=request_body, model=model)
)
if counted is not None:
return counted
@ -1754,11 +1629,7 @@ def _estimate_output_tokens(
request_body: dict,
route: str,
model_info: Mapping[str, object],
decision_requests: Sequence[tuple[str, DecisionRerankRequest]] = (),
) -> int | None:
if decision_requests:
per_call: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK
return per_call * max(request.call_count for _, request in decision_requests)
if _is_input_only_route(route=route):
return 0

View file

@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
tool_choice=request_body.get("tool_choice"),
)
except ValueError:
return count_text_tokens(model=model, text=request_body.get("messages"))
return _count_text_tokens(model=model, text=request_body.get("messages"))
if "prompt" in request_body:
return count_text_tokens(model=model, text=request_body.get("prompt"))
return _count_text_tokens(model=model, text=request_body.get("prompt"))
if "input" in request_body:
return count_text_tokens(
return _count_text_tokens(
model=model, text=billable_input_since_newest_compaction(request_body.get("input"))
)
if "query" in request_body or "documents" in request_body:
query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query"))
document_tokens: Final = count_text_tokens(
query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query"))
document_tokens: Final = _count_text_tokens(
model=model,
text=request_body.get("documents"),
)
@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
return None
def count_text_tokens(model: str, text: object) -> int:
def _count_text_tokens(model: str, text: object) -> int:
if text is None:
return 0

View file

@ -149,6 +149,10 @@ def _finish(
return response
def _call_count(request: DecisionRerankRequest) -> int:
return 1 if request.strategy == "multiple_choice" else len(request.documents)
def _rerank(
prepare: Callable[[int], _DocumentCall],
request: DecisionRerankRequest,
@ -157,7 +161,7 @@ def _rerank(
provider: str,
deadline: float,
) -> DecisionRerankResponse:
count: Final = request.call_count
count: Final = _call_count(request)
workers: Final = min(count, _MAX_CONCURRENCY)
completed: Final[SimpleQueue[DecisionRerankResult]] = SimpleQueue()
@ -193,7 +197,7 @@ async def _arerank(
provider: str,
deadline: float,
) -> DecisionRerankResponse:
count: Final = request.call_count
count: Final = _call_count(request)
workers: Final = min(count, _MAX_CONCURRENCY)
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
cancellation_requested: Final = asyncio.Event()

View file

@ -66,10 +66,6 @@ class DecisionRerankRequest(BaseModel):
document: Final = self.documents[index]
return document if isinstance(document, str) else document.text
@property
def call_count(self) -> int:
return 1 if self.strategy == "multiple_choice" else len(self.documents)
@dataclass(frozen=True, slots=True)
class DecisionRerankResult:

View file

@ -25,13 +25,11 @@ from litellm.proxy.common_utils.user_api_key_cache import (
)
from litellm.proxy.spend_tracking.budget_reservation import (
_get_team_member_budget_counter,
count_request_input_tokens,
estimate_request_max_cost,
get_budget_window_start,
release_unbound_budget_reservation,
reserve_budget_for_request,
)
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -222,140 +220,6 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
assert math.isclose(after, before * 1000)
def _rerank_router(backend_model: str, custom_llm_provider: str | None = None, api_base: str | None = None) -> Router:
return Router(
model_list=[
Deployment(
model_name="budgeted-rerank",
litellm_params=LiteLLM_Params(
model=backend_model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key="sk-fake"
),
model_info=ModelInfo(
id="budgeted-rerank-deployment",
input_cost_per_token=0.001,
output_cost_per_token=0.002,
max_output_tokens=10,
),
).model_dump()
]
)
RERANK_QUERY: Final = "relevant query " * 100
RERANK_BODY: Final = {
"model": "budgeted-rerank",
"query": RERANK_QUERY,
"documents": ["first passage", {"text": "second passage"}, "third passage", "fourth passage"],
"max_tokens": 1,
}
async def _reserve_rerank(router: Router, max_budget: float) -> dict[str, object] | None:
return await reserve_budget_for_request(
request_body=RERANK_BODY,
route="/v1/rerank",
llm_router=router,
valid_token=UserAPIKeyAuth(token="decision-rerank-budget", max_budget=max_budget, spend=0.0),
team_object=None,
user_object=None,
prisma_client=None,
user_api_key_cache=UserApiKeyCache(),
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
fail_closed_budget_enforcement=True,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("backend_model", "call_count"),
[("typesafe/jev-latest", 4), ("perplexity/pplx-decider-latest", 1)],
)
async def test_decision_rerank_reserves_transformed_inputs_and_output_for_every_call(
spend_counter_cache: DualCache, backend_model: str, call_count: int
) -> None:
reservation: Final = await _reserve_rerank(_rerank_router(backend_model), max_budget=100.0)
query_tokens: Final = litellm.token_counter(model=backend_model, text=RERANK_QUERY)
assert reservation is not None
input_tokens: Final = reservation["input_tokens"]
assert isinstance(input_tokens, int)
assert input_tokens > query_tokens * call_count
assert reservation["input_cost"] == pytest.approx(input_tokens * 0.001)
assert reservation["reserved_cost"] == pytest.approx(input_tokens * 0.001 + call_count * 10 * 0.002)
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx(
reservation["reserved_cost"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("backend_model", ["typesafe/jev-latest", "perplexity/pplx-decider-latest"])
async def test_decision_rerank_rejects_a_budget_that_only_covers_the_untransformed_request(
spend_counter_cache: DualCache, backend_model: str
) -> None:
raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank")
assert raw_tokens is not None
with pytest.raises(litellm.BudgetExceededError, match="Estimated request cost"):
await _reserve_rerank(_rerank_router(backend_model), max_budget=raw_tokens * 0.001 + 0.001)
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx(0)
def test_native_rerank_remains_input_only() -> None:
router: Final = _rerank_router("cohere/rerank-v3.5")
raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank")
assert raw_tokens is not None
estimate: Final = estimate_request_max_cost(request_body=RERANK_BODY, route="/v1/rerank", llm_router=router)
assert estimate == pytest.approx(raw_tokens * 0.001)
@pytest.mark.parametrize("invalid_fields", [{"documents": []}, {"top_n": 0}])
def test_decision_rerank_admission_preserves_client_validation_errors(invalid_fields: dict[str, object]) -> None:
with pytest.raises(litellm.BadRequestError) as error:
estimate_request_max_cost(
request_body={**RERANK_BODY, **invalid_fields},
route="/v1/rerank",
llm_router=_rerank_router("typesafe/jev-latest"),
)
assert error.value.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "provider", "api_base"),
[
("jev-latest", "typesafe", None),
("pplx-decider-latest", "perplexity", None),
("pplx-decider-latest", "perplexity", "https://api.perplexity.ai"),
],
)
@pytest.mark.parametrize("use_router", [False, True])
async def test_decision_rerank_provider_configuration_reserves_the_same_cost_as_provider_prefix(
model: str, provider: str, api_base: str | None, use_router: bool
) -> None:
prefixed_model: Final = f"{provider}/{model}"
prefixed_router: Final = _rerank_router(prefixed_model) if use_router else None
explicit_provider: Final = None if api_base is not None else provider
explicit_router: Final = _rerank_router(model, explicit_provider, api_base) if use_router else None
prefixed_body: Final = RERANK_BODY if use_router else {**RERANK_BODY, "model": prefixed_model}
explicit_body: Final = (
RERANK_BODY
if use_router
else {**RERANK_BODY, "model": model, "custom_llm_provider": explicit_provider, "api_base": api_base}
)
prefixed_counts: Final = await count_request_input_tokens(prefixed_body, "/v1/rerank", prefixed_router)
explicit_counts: Final = await count_request_input_tokens(explicit_body, "/v1/rerank", explicit_router)
prefixed_cost: Final = estimate_request_max_cost(prefixed_body, "/v1/rerank", prefixed_router, prefixed_counts)
explicit_cost: Final = estimate_request_max_cost(explicit_body, "/v1/rerank", explicit_router, explicit_counts)
assert prefixed_cost is not None and prefixed_cost > 0
assert explicit_cost == pytest.approx(prefixed_cost)
assert tuple(explicit_counts.values()) == tuple(prefixed_counts.values())
@pytest.mark.asyncio
@pytest.mark.parametrize(
"expiry_offset, expected_max_budget",