fix(proxy): hold Gemini candidates of a model with no context window up to the default output

A Gemini model the cost map has no context window for, priced in its deployment, held one
candidate for any candidateCount, so parallel calls could pass their budget holds. Its
candidates are held again up to its counted prompt plus the default output hold
This commit is contained in:
mateo-berri 2026-10-30 08:09:18 -07:00
parent 6bb3590643
commit a2f15098f7
2 changed files with 30 additions and 9 deletions

View file

@ -1465,20 +1465,18 @@ def _max_cost_for_cost_info(
held_output_tokens=output_tokens,
output_tokens=output_tokens,
)
main_hold: Final = _main_generate_content_hold(model_info)
if one_candidate_hold is None or main_hold is None:
hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens)
if one_candidate_hold is None or hold_ceiling is None:
return one_candidate_hold
return max(one_candidate_hold, min(candidates_hold, main_hold))
return max(one_candidate_hold, min(candidates_hold, hold_ceiling))
def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None:
def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None:
context_window: Final = _to_int(model_info.get("max_input_tokens"))
if context_window is None:
return None
default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info))
return _priced_hold(
model_info=model_info,
input_tokens=context_window,
input_tokens=counted_input_tokens if context_window is None else context_window,
held_output_tokens=default_output_tokens,
output_tokens=default_output_tokens,
)

View file

@ -381,14 +381,17 @@ LOW_OUTPUT_CEILING: Final = {**WIDE_WINDOW, "max_output_tokens": 1000}
def _priced_gemini_hold(
model_info: dict[str, object], generation_config: dict[str, object], input_tokens: int = 10
model_info: dict[str, object],
generation_config: dict[str, object],
input_tokens: int = 10,
model: str = "vertex_ai/gemini-3.8-flash",
) -> float | None:
router: Final = Router(
model_list=[
{
"model_name": "assistant",
"litellm_params": {
"model": "vertex_ai/gemini-3.8-flash",
"model": model,
"input_cost_per_token": 1e-6,
"output_cost_per_token": 4e-6,
},
@ -447,6 +450,26 @@ def test_candidates_never_hold_less_than_one_candidate() -> None:
assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold
NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry"
@pytest.mark.parametrize(
("generation_config", "same_hold_config"),
(
({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}),
({"maxOutputTokens": 8000, "candidateCount": 4}, {}),
({"candidateCount": 1_000_000}, {}),
),
)
def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate(
generation_config: dict[str, object], same_hold_config: dict[str, object]
) -> None:
no_window: Final = {"max_output_tokens": 100_000}
expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL)
assert expected_hold is not None
assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
def test_an_n_too_large_to_price_holds_one_choice(count: object) -> None:
one_choice_hold: Final = _chat_hold(GEMINI_CHAT_BODY)