From 55692c25dc17a079ba376f7d8510598fa5ebaaf9 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Fri, 23 Oct 2026 12:11:02 -0700 Subject: [PATCH] fix(rerank): keep shared budget reservation unchanged Restore shared proxy reservation and token counting to main, remove the associated reservation tests, and restore the original rerank call-count helper Keep rerank response cost aggregation and explicit duration pricing Validation passed with 748 focused tests and make check --- .../spend_tracking/budget_reservation.py | 161 ++---------------- litellm/proxy/spend_tracking/input_tokens.py | 12 +- .../handler.py | 8 +- .../transformation.py | 4 - .../spend_tracking/test_budget_reservation.py | 136 --------------- 5 files changed, 28 insertions(+), 293 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f39b06eb83a..803df9f83ca 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -39,18 +39,9 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import ( - count_input_tokens, - count_input_tokens_for_model, - count_text_tokens, -) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging -from litellm.rerank_api.litellm_decisions_transformation.transformation import ( - DecisionRerankRequest, - LiteLLMDecisionsRerankConfig, -) -from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -83,7 +74,6 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { _CACHED_VALUE: Final = TypeAdapter(object) _WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_REQUEST_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) class _CounterReservationUnavailable(Exception): @@ -1351,34 +1341,26 @@ def _estimate_request_input_cost_for_model( llm_router: Router | None, input_tokens: int | None = None, ) -> float | None: - body: Final = _REQUEST_BODY.validate_python(request_body) estimates: Final = [ _input_cost_for_cost_info( - request_body=body, + request_body=request_body, route=route, model=model, model_info=model_info, input_tokens=input_tokens, - llm_router=llm_router, - ) - for model_info in _get_model_cost_infos( - model=model, - llm_router=llm_router, - custom_llm_provider=_request_string(body, "custom_llm_provider"), - api_base=_request_string(body, "api_base"), ) + for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) ] valid_estimates: Final = [estimate for estimate in estimates if estimate is not None] return max(valid_estimates) if valid_estimates else None def _input_cost_for_cost_info( - request_body: dict[str, object], + request_body: dict, route: str, model: str, model_info: Mapping[str, object], input_tokens: int | None = None, - llm_router: Router | None = None, ) -> float | None: estimated_input_tokens: Final = _estimate_input_tokens( request_body=request_body, @@ -1386,7 +1368,6 @@ def _input_cost_for_cost_info( model=model, model_info=model_info, input_tokens=input_tokens, - llm_router=llm_router, ) if estimated_input_tokens is None: return None @@ -1408,34 +1389,26 @@ def _estimate_request_max_cost_for_model( llm_router: Router | None, input_tokens: int | None = None, ) -> float | None: - body: Final = _REQUEST_BODY.validate_python(request_body) estimates: Final = [ _max_cost_for_cost_info( - request_body=body, + request_body=request_body, route=route, model=model, model_info=model_info, input_tokens=input_tokens, - llm_router=llm_router, - ) - for model_info in _get_model_cost_infos( - model=model, - llm_router=llm_router, - custom_llm_provider=_request_string(body, "custom_llm_provider"), - api_base=_request_string(body, "api_base"), ) + for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) ] valid_estimates: Final = [estimate for estimate in estimates if estimate is not None] return max(valid_estimates) if valid_estimates else None def _max_cost_for_cost_info( - request_body: dict[str, object], + request_body: dict, route: str, model: str, model_info: Mapping[str, object], input_tokens: int | None = None, - llm_router: Router | None = None, ) -> float | None: image_cost: Final = _estimate_image_generation_cost( request_body=request_body, @@ -1450,13 +1423,11 @@ def _max_cost_for_cost_info( model=model, model_info=model_info, input_tokens=input_tokens, - llm_router=llm_router, ) output_tokens: Final = _estimate_output_tokens( request_body=request_body, route=route, model_info=model_info, - decision_requests=_decision_rerank_requests(request_body, route, model, llm_router), ) if estimated_input_tokens is None or output_tokens is None: return None @@ -1536,26 +1507,17 @@ def _estimate_image_generation_cost( def _get_model_cost_info( model: str, llm_router: Router | None, - custom_llm_provider: str | None = None, - api_base: str | None = None, ) -> Mapping[str, object] | None: if llm_router is not None: model_group_info: Final = llm_router.cached_model_group_info(model) if model_group_info is not None: return model_group_info.model_dump() - resolved_model, provider, _, _ = ( - litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) - if api_base is not None - else (model, custom_llm_provider, None, None) - ) - return dict(litellm.get_model_info(model=resolved_model, custom_llm_provider=provider, api_base=api_base)) + return dict(litellm.get_model_info(model=model)) def _get_model_cost_infos( model: str, llm_router: Router | None, - custom_llm_provider: str | None = None, - api_base: str | None = None, ) -> Sequence[Mapping[str, object]]: """Cost-info candidates to estimate a request against for one model group. @@ -1566,9 +1528,7 @@ def _get_model_cost_infos( deployment can never leave the request under-reserved. """ try: - base: Final = _get_model_cost_info( - model=model, llm_router=llm_router, custom_llm_provider=custom_llm_provider, api_base=api_base - ) + base: Final = _get_model_cost_info(model=model, llm_router=llm_router) if base is None: return [] tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router) @@ -1634,109 +1594,24 @@ async def count_request_input_tokens( """Input-token count per candidate model, counted once per request. The counts are reused by both the max-cost and the input-cost estimate.""" - body: Final = _REQUEST_BODY.validate_python(request_body) models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - decision_requests: Final = { - model: requests for model in models if (requests := _decision_rerank_requests(body, route, model, llm_router)) - } - ordinary_counts: Final = await count_input_tokens( - request_body=request_body, - raw_body=raw_body, - models=models, - ) - decision_counts: Final = await asyncio.gather( - *(asyncio.to_thread(_decision_rerank_input_tokens, requests) for requests in decision_requests.values()) - ) - return MappingProxyType( - { - **ordinary_counts, - **{ - model: max(tokens, ordinary_counts.get(model, 0)) - for model, tokens in zip(decision_requests, decision_counts, strict=True) - }, - } - ) - - -def _request_string(request_body: Mapping[str, object], field: str) -> str | None: - value: Final = request_body.get(field) - return value if isinstance(value, str) else None - - -def _decision_rerank_request( - request_body: Mapping[str, object], model: str, custom_llm_provider: str | None, api_base: str | None -) -> tuple[str, DecisionRerankRequest] | None: - try: - resolved_model, provider, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base - ) - except litellm.BadRequestError: - return None - strategy: Final = get_decision_rerank_strategy(resolved_model, provider) - if strategy is None: - return None - return ( - f"{provider}/{resolved_model}", - LiteLLMDecisionsRerankConfig.validate_request(request_body, model=model, provider=provider, strategy=strategy), - ) - - -def _decision_rerank_requests( - request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None -) -> tuple[tuple[str, DecisionRerankRequest], ...]: - if "rerank" not in route: - return () - deployments: Final = llm_router.get_model_list(model_name=model) if llm_router is not None else () - models: Final = tuple( - dict.fromkeys( - ( - deployment["litellm_params"]["model"], - _request_string(deployment["litellm_params"], "custom_llm_provider"), - _request_string(deployment["litellm_params"], "api_base"), - ) - for deployment in deployments or () - if "model" in deployment["litellm_params"] - ) - ) - return tuple( - request - for backend_model, provider, api_base in models - or ((model, _request_string(request_body, "custom_llm_provider"), _request_string(request_body, "api_base")),) - if (request := _decision_rerank_request(request_body, backend_model, provider, api_base)) is not None - ) - - -def _decision_rerank_input_tokens(requests: Sequence[tuple[str, DecisionRerankRequest]]) -> int: - return max( - sum( - count_text_tokens( - model=model, - text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json( - exclude_none=True, exclude_defaults=True - ), - ) - for index in range(request.call_count) - ) - for model, request in requests - ) + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( - request_body: dict[str, object], + request_body: dict, route: str, model: str, model_info: Mapping[str, object], input_tokens: int | None = None, - llm_router: Router | None = None, ) -> int | None: - if input_tokens is not None: - return input_tokens - decision_requests: Final = _decision_rerank_requests(request_body, route, model, llm_router) - if decision_requests: - return _decision_rerank_input_tokens(decision_requests) - counted: Final = count_input_tokens_for_model(request_body=request_body, model=model) + counted: Final = ( + input_tokens + if input_tokens is not None + else count_input_tokens_for_model(request_body=request_body, model=model) + ) if counted is not None: return counted @@ -1754,11 +1629,7 @@ def _estimate_output_tokens( request_body: dict, route: str, model_info: Mapping[str, object], - decision_requests: Sequence[tuple[str, DecisionRerankRequest]] = (), ) -> int | None: - if decision_requests: - per_call: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK - return per_call * max(request.call_count for _, request in decision_requests) if _is_input_only_route(route=route): return 0 diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 763807dd52a..927152296e9 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: tool_choice=request_body.get("tool_choice"), ) except ValueError: - return count_text_tokens(model=model, text=request_body.get("messages")) + return _count_text_tokens(model=model, text=request_body.get("messages")) if "prompt" in request_body: - return count_text_tokens(model=model, text=request_body.get("prompt")) + return _count_text_tokens(model=model, text=request_body.get("prompt")) if "input" in request_body: - return count_text_tokens( + return _count_text_tokens( model=model, text=billable_input_since_newest_compaction(request_body.get("input")) ) if "query" in request_body or "documents" in request_body: - query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = count_text_tokens( + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( model=model, text=request_body.get("documents"), ) @@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: return None -def count_text_tokens(model: str, text: object) -> int: +def _count_text_tokens(model: str, text: object) -> int: if text is None: return 0 diff --git a/litellm/rerank_api/litellm_decisions_transformation/handler.py b/litellm/rerank_api/litellm_decisions_transformation/handler.py index 14e1be02b83..366935f4977 100644 --- a/litellm/rerank_api/litellm_decisions_transformation/handler.py +++ b/litellm/rerank_api/litellm_decisions_transformation/handler.py @@ -149,6 +149,10 @@ def _finish( return response +def _call_count(request: DecisionRerankRequest) -> int: + return 1 if request.strategy == "multiple_choice" else len(request.documents) + + def _rerank( prepare: Callable[[int], _DocumentCall], request: DecisionRerankRequest, @@ -157,7 +161,7 @@ def _rerank( provider: str, deadline: float, ) -> DecisionRerankResponse: - count: Final = request.call_count + count: Final = _call_count(request) workers: Final = min(count, _MAX_CONCURRENCY) completed: Final[SimpleQueue[DecisionRerankResult]] = SimpleQueue() @@ -193,7 +197,7 @@ async def _arerank( provider: str, deadline: float, ) -> DecisionRerankResponse: - count: Final = request.call_count + count: Final = _call_count(request) workers: Final = min(count, _MAX_CONCURRENCY) outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing cancellation_requested: Final = asyncio.Event() diff --git a/litellm/rerank_api/litellm_decisions_transformation/transformation.py b/litellm/rerank_api/litellm_decisions_transformation/transformation.py index 352151524d0..022781d0d1e 100644 --- a/litellm/rerank_api/litellm_decisions_transformation/transformation.py +++ b/litellm/rerank_api/litellm_decisions_transformation/transformation.py @@ -66,10 +66,6 @@ class DecisionRerankRequest(BaseModel): document: Final = self.documents[index] return document if isinstance(document, str) else document.text - @property - def call_count(self) -> int: - return 1 if self.strategy == "multiple_choice" else len(self.documents) - @dataclass(frozen=True, slots=True) class DecisionRerankResult: diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 8e9c1c17565..024fe5229be 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -25,13 +25,11 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, - count_request_input_tokens, estimate_request_max_cost, get_budget_window_start, release_unbound_budget_reservation, reserve_budget_for_request, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -222,140 +220,6 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) -def _rerank_router(backend_model: str, custom_llm_provider: str | None = None, api_base: str | None = None) -> Router: - return Router( - model_list=[ - Deployment( - model_name="budgeted-rerank", - litellm_params=LiteLLM_Params( - model=backend_model, custom_llm_provider=custom_llm_provider, api_base=api_base, api_key="sk-fake" - ), - model_info=ModelInfo( - id="budgeted-rerank-deployment", - input_cost_per_token=0.001, - output_cost_per_token=0.002, - max_output_tokens=10, - ), - ).model_dump() - ] - ) - - -RERANK_QUERY: Final = "relevant query " * 100 -RERANK_BODY: Final = { - "model": "budgeted-rerank", - "query": RERANK_QUERY, - "documents": ["first passage", {"text": "second passage"}, "third passage", "fourth passage"], - "max_tokens": 1, -} - - -async def _reserve_rerank(router: Router, max_budget: float) -> dict[str, object] | None: - return await reserve_budget_for_request( - request_body=RERANK_BODY, - route="/v1/rerank", - llm_router=router, - valid_token=UserAPIKeyAuth(token="decision-rerank-budget", max_budget=max_budget, spend=0.0), - team_object=None, - user_object=None, - prisma_client=None, - user_api_key_cache=UserApiKeyCache(), - proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), - fail_closed_budget_enforcement=True, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("backend_model", "call_count"), - [("typesafe/jev-latest", 4), ("perplexity/pplx-decider-latest", 1)], -) -async def test_decision_rerank_reserves_transformed_inputs_and_output_for_every_call( - spend_counter_cache: DualCache, backend_model: str, call_count: int -) -> None: - reservation: Final = await _reserve_rerank(_rerank_router(backend_model), max_budget=100.0) - query_tokens: Final = litellm.token_counter(model=backend_model, text=RERANK_QUERY) - - assert reservation is not None - input_tokens: Final = reservation["input_tokens"] - assert isinstance(input_tokens, int) - assert input_tokens > query_tokens * call_count - assert reservation["input_cost"] == pytest.approx(input_tokens * 0.001) - assert reservation["reserved_cost"] == pytest.approx(input_tokens * 0.001 + call_count * 10 * 0.002) - assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx( - reservation["reserved_cost"] - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("backend_model", ["typesafe/jev-latest", "perplexity/pplx-decider-latest"]) -async def test_decision_rerank_rejects_a_budget_that_only_covers_the_untransformed_request( - spend_counter_cache: DualCache, backend_model: str -) -> None: - raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank") - assert raw_tokens is not None - - with pytest.raises(litellm.BudgetExceededError, match="Estimated request cost"): - await _reserve_rerank(_rerank_router(backend_model), max_budget=raw_tokens * 0.001 + 0.001) - - assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-rerank-budget") == pytest.approx(0) - - -def test_native_rerank_remains_input_only() -> None: - router: Final = _rerank_router("cohere/rerank-v3.5") - raw_tokens: Final = count_input_tokens_for_model(request_body=RERANK_BODY, model="budgeted-rerank") - assert raw_tokens is not None - - estimate: Final = estimate_request_max_cost(request_body=RERANK_BODY, route="/v1/rerank", llm_router=router) - - assert estimate == pytest.approx(raw_tokens * 0.001) - - -@pytest.mark.parametrize("invalid_fields", [{"documents": []}, {"top_n": 0}]) -def test_decision_rerank_admission_preserves_client_validation_errors(invalid_fields: dict[str, object]) -> None: - with pytest.raises(litellm.BadRequestError) as error: - estimate_request_max_cost( - request_body={**RERANK_BODY, **invalid_fields}, - route="/v1/rerank", - llm_router=_rerank_router("typesafe/jev-latest"), - ) - - assert error.value.status_code == 400 - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("model", "provider", "api_base"), - [ - ("jev-latest", "typesafe", None), - ("pplx-decider-latest", "perplexity", None), - ("pplx-decider-latest", "perplexity", "https://api.perplexity.ai"), - ], -) -@pytest.mark.parametrize("use_router", [False, True]) -async def test_decision_rerank_provider_configuration_reserves_the_same_cost_as_provider_prefix( - model: str, provider: str, api_base: str | None, use_router: bool -) -> None: - prefixed_model: Final = f"{provider}/{model}" - prefixed_router: Final = _rerank_router(prefixed_model) if use_router else None - explicit_provider: Final = None if api_base is not None else provider - explicit_router: Final = _rerank_router(model, explicit_provider, api_base) if use_router else None - prefixed_body: Final = RERANK_BODY if use_router else {**RERANK_BODY, "model": prefixed_model} - explicit_body: Final = ( - RERANK_BODY - if use_router - else {**RERANK_BODY, "model": model, "custom_llm_provider": explicit_provider, "api_base": api_base} - ) - prefixed_counts: Final = await count_request_input_tokens(prefixed_body, "/v1/rerank", prefixed_router) - explicit_counts: Final = await count_request_input_tokens(explicit_body, "/v1/rerank", explicit_router) - prefixed_cost: Final = estimate_request_max_cost(prefixed_body, "/v1/rerank", prefixed_router, prefixed_counts) - explicit_cost: Final = estimate_request_max_cost(explicit_body, "/v1/rerank", explicit_router, explicit_counts) - - assert prefixed_cost is not None and prefixed_cost > 0 - assert explicit_cost == pytest.approx(prefixed_cost) - assert tuple(explicit_counts.values()) == tuple(prefixed_counts.values()) - - @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget",