From a2f15098f784cd120856d7d1bea2d96a0736655e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:09:18 -0700 Subject: [PATCH] fix(proxy): hold Gemini candidates of a model with no context window up to the default output A Gemini model the cost map has no context window for, priced in its deployment, held one candidate for any candidateCount, so parallel calls could pass their budget holds. Its candidates are held again up to its counted prompt plus the default output hold --- .../spend_tracking/budget_reservation.py | 12 ++++----- .../spend_tracking/test_budget_reservation.py | 27 +++++++++++++++++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f45f9028994..a2e0337ce02 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1465,20 +1465,18 @@ def _max_cost_for_cost_info( held_output_tokens=output_tokens, output_tokens=output_tokens, ) - main_hold: Final = _main_generate_content_hold(model_info) - if one_candidate_hold is None or main_hold is None: + hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens) + if one_candidate_hold is None or hold_ceiling is None: return one_candidate_hold - return max(one_candidate_hold, min(candidates_hold, main_hold)) + return max(one_candidate_hold, min(candidates_hold, hold_ceiling)) -def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None: +def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None: context_window: Final = _to_int(model_info.get("max_input_tokens")) - if context_window is None: - return None default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info)) return _priced_hold( model_info=model_info, - input_tokens=context_window, + input_tokens=counted_input_tokens if context_window is None else context_window, held_output_tokens=default_output_tokens, output_tokens=default_output_tokens, ) diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index f3227130974..5baddde6b39 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -381,14 +381,17 @@ LOW_OUTPUT_CEILING: Final = {**WIDE_WINDOW, "max_output_tokens": 1000} def _priced_gemini_hold( - model_info: dict[str, object], generation_config: dict[str, object], input_tokens: int = 10 + model_info: dict[str, object], + generation_config: dict[str, object], + input_tokens: int = 10, + model: str = "vertex_ai/gemini-3.8-flash", ) -> float | None: router: Final = Router( model_list=[ { "model_name": "assistant", "litellm_params": { - "model": "vertex_ai/gemini-3.8-flash", + "model": model, "input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, }, @@ -447,6 +450,26 @@ def test_candidates_never_hold_less_than_one_candidate() -> None: assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold +NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry" + + +@pytest.mark.parametrize( + ("generation_config", "same_hold_config"), + ( + ({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}), + ({"maxOutputTokens": 8000, "candidateCount": 4}, {}), + ({"candidateCount": 1_000_000}, {}), + ), +) +def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate( + generation_config: dict[str, object], same_hold_config: dict[str, object] +) -> None: + no_window: Final = {"max_output_tokens": 100_000} + expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL) + assert expected_hold is not None + assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold + + @pytest.mark.parametrize("count", UNPRICEABLE_COUNTS) def test_an_n_too_large_to_price_holds_one_choice(count: object) -> None: one_choice_hold: Final = _chat_hold(GEMINI_CHAT_BODY)