diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f45f9028994..a2e0337ce02 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1465,20 +1465,18 @@ def _max_cost_for_cost_info( held_output_tokens=output_tokens, output_tokens=output_tokens, ) - main_hold: Final = _main_generate_content_hold(model_info) - if one_candidate_hold is None or main_hold is None: + hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens) + if one_candidate_hold is None or hold_ceiling is None: return one_candidate_hold - return max(one_candidate_hold, min(candidates_hold, main_hold)) + return max(one_candidate_hold, min(candidates_hold, hold_ceiling)) -def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None: +def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None: context_window: Final = _to_int(model_info.get("max_input_tokens")) - if context_window is None: - return None default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info)) return _priced_hold( model_info=model_info, - input_tokens=context_window, + input_tokens=counted_input_tokens if context_window is None else context_window, held_output_tokens=default_output_tokens, output_tokens=default_output_tokens, ) diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index f3227130974..5baddde6b39 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -381,14 +381,17 @@ LOW_OUTPUT_CEILING: Final = {**WIDE_WINDOW, "max_output_tokens": 1000} def _priced_gemini_hold( - model_info: dict[str, object], generation_config: dict[str, object], input_tokens: int = 10 + model_info: dict[str, object], + generation_config: dict[str, object], + input_tokens: int = 10, + model: str = "vertex_ai/gemini-3.8-flash", ) -> float | None: router: Final = Router( model_list=[ { "model_name": "assistant", "litellm_params": { - "model": "vertex_ai/gemini-3.8-flash", + "model": model, "input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6, }, @@ -447,6 +450,26 @@ def test_candidates_never_hold_less_than_one_candidate() -> None: assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold +NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry" + + +@pytest.mark.parametrize( + ("generation_config", "same_hold_config"), + ( + ({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}), + ({"maxOutputTokens": 8000, "candidateCount": 4}, {}), + ({"candidateCount": 1_000_000}, {}), + ), +) +def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate( + generation_config: dict[str, object], same_hold_config: dict[str, object] +) -> None: + no_window: Final = {"max_output_tokens": 100_000} + expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL) + assert expected_hold is not None + assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold + + @pytest.mark.parametrize("count", UNPRICEABLE_COUNTS) def test_an_n_too_large_to_price_holds_one_choice(count: object) -> None: one_choice_hold: Final = _chat_hold(GEMINI_CHAT_BODY)