mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): hold Gemini candidates of a model with no context window up to the default output
A Gemini model the cost map has no context window for, priced in its deployment, held one candidate for any candidateCount, so parallel calls could pass their budget holds. Its candidates are held again up to its counted prompt plus the default output hold
This commit is contained in:
parent
6bb3590643
commit
a2f15098f7
2 changed files with 30 additions and 9 deletions
|
|
@ -1465,20 +1465,18 @@ def _max_cost_for_cost_info(
|
|||
held_output_tokens=output_tokens,
|
||||
output_tokens=output_tokens,
|
||||
)
|
||||
main_hold: Final = _main_generate_content_hold(model_info)
|
||||
if one_candidate_hold is None or main_hold is None:
|
||||
hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens)
|
||||
if one_candidate_hold is None or hold_ceiling is None:
|
||||
return one_candidate_hold
|
||||
return max(one_candidate_hold, min(candidates_hold, main_hold))
|
||||
return max(one_candidate_hold, min(candidates_hold, hold_ceiling))
|
||||
|
||||
|
||||
def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None:
|
||||
def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None:
|
||||
context_window: Final = _to_int(model_info.get("max_input_tokens"))
|
||||
if context_window is None:
|
||||
return None
|
||||
default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info))
|
||||
return _priced_hold(
|
||||
model_info=model_info,
|
||||
input_tokens=context_window,
|
||||
input_tokens=counted_input_tokens if context_window is None else context_window,
|
||||
held_output_tokens=default_output_tokens,
|
||||
output_tokens=default_output_tokens,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -381,14 +381,17 @@ LOW_OUTPUT_CEILING: Final = {**WIDE_WINDOW, "max_output_tokens": 1000}
|
|||
|
||||
|
||||
def _priced_gemini_hold(
|
||||
model_info: dict[str, object], generation_config: dict[str, object], input_tokens: int = 10
|
||||
model_info: dict[str, object],
|
||||
generation_config: dict[str, object],
|
||||
input_tokens: int = 10,
|
||||
model: str = "vertex_ai/gemini-3.8-flash",
|
||||
) -> float | None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "assistant",
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/gemini-3.8-flash",
|
||||
"model": model,
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
},
|
||||
|
|
@ -447,6 +450,26 @@ def test_candidates_never_hold_less_than_one_candidate() -> None:
|
|||
assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold
|
||||
|
||||
|
||||
NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("generation_config", "same_hold_config"),
|
||||
(
|
||||
({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}),
|
||||
({"maxOutputTokens": 8000, "candidateCount": 4}, {}),
|
||||
({"candidateCount": 1_000_000}, {}),
|
||||
),
|
||||
)
|
||||
def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate(
|
||||
generation_config: dict[str, object], same_hold_config: dict[str, object]
|
||||
) -> None:
|
||||
no_window: Final = {"max_output_tokens": 100_000}
|
||||
expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL)
|
||||
assert expected_hold is not None
|
||||
assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold
|
||||
|
||||
|
||||
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
|
||||
def test_an_n_too_large_to_price_holds_one_choice(count: object) -> None:
|
||||
one_choice_hold: Final = _chat_hold(GEMINI_CHAT_BODY)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue