fix(proxy): keep main's Gemini hold unless the request caps its output

An uncapped generateContent call, or one capped only by a snake_case
spelling, now holds what main holds (the context window plus the default
output). A call with generationConfig.maxOutputTokens holds its counted
prompt plus cap times candidateCount, never clamped to the deployment's
max_output_tokens and never more than main's hold. A model with no context
window keeps main's no-hold, and candidateCount always counts on
generateContent routes. Also keeps get_litellm_virtual_key in
pass_through_endpoints/common_utils.py, so an out-of-tree importer
does not break
This commit is contained in:
mateo-berri 2026-10-30 21:23:23 -07:00
parent 7df5e93f68
commit 82ac5a1ef9
4 changed files with 156 additions and 123 deletions

View file

@ -0,0 +1,18 @@
from typing import Final
from fastapi import Request
def get_litellm_virtual_key(request: Request) -> str:
"""
Extract and format API key from request headers.
Prioritizes x-litellm-api-key over Authorization header.
Vertex JS SDK uses `Authorization` header, we use `x-litellm-api-key` to pass litellm virtual key
"""
litellm_api_key: Final = request.headers.get("x-litellm-api-key")
if litellm_api_key:
return f"Bearer {litellm_api_key}"
return request.headers.get("Authorization", "")

View file

@ -55,7 +55,6 @@ from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
from litellm.types.utils import CallTypes
from litellm.utils import ProviderConfigManager
@dataclass
@ -1404,9 +1403,7 @@ def _estimate_request_max_cost_for_model(
llm_router: Router | None,
input_tokens: int | None = None,
) -> float | None:
output_multiplier: Final = _get_output_multiplier(
_requested_output_counts(request_body=request_body, route=route, model=model, llm_router=llm_router)
)
output_multiplier: Final = _get_output_multiplier(_requested_output_counts(request_body=request_body, route=route))
estimates: Final = [
_max_cost_for_cost_info(
request_body=request_body,
@ -1444,39 +1441,54 @@ def _max_cost_for_cost_info(
model_info=model_info,
input_tokens=input_tokens,
)
if estimated_input_tokens is None:
return None
if RouteChecks.is_generate_content_route(route):
return _generate_content_hold(
request_body=request_body,
model_info=model_info,
input_tokens=estimated_input_tokens,
output_multiplier=output_multiplier,
)
output_tokens: Final = _estimate_output_tokens(
request_body=request_body,
route=route,
model_info=model_info,
)
if estimated_input_tokens is None or output_tokens is None:
if output_tokens is None:
return None
candidates_hold: Final = _priced_hold(
return _priced_hold(
model_info=model_info,
input_tokens=estimated_input_tokens,
held_output_tokens=output_tokens * output_multiplier,
output_tokens=output_tokens,
)
if candidates_hold is None or output_multiplier == 1 or not RouteChecks.is_generate_content_route(route):
return candidates_hold
one_candidate_hold: Final = _priced_hold(
def _generate_content_hold(
request_body: Mapping[str, object], model_info: Mapping[str, object], input_tokens: int, output_multiplier: int
) -> float | None:
main_hold: Final = _main_generate_content_hold(model_info)
requested_output_tokens: Final = _gemini_requested_output_tokens(request_body)
if main_hold is None or requested_output_tokens is None:
return main_hold
capped_hold: Final = _priced_hold(
model_info=model_info,
input_tokens=estimated_input_tokens,
held_output_tokens=output_tokens,
output_tokens=output_tokens,
input_tokens=input_tokens,
held_output_tokens=requested_output_tokens * output_multiplier,
output_tokens=requested_output_tokens,
)
hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens)
if one_candidate_hold is None or hold_ceiling is None:
return one_candidate_hold
return max(one_candidate_hold, min(candidates_hold, hold_ceiling))
return main_hold if capped_hold is None else min(capped_hold, main_hold)
def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None:
def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None:
context_window: Final = _to_int(model_info.get("max_input_tokens"))
if context_window is None:
return None
default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info))
return _priced_hold(
model_info=model_info,
input_tokens=counted_input_tokens if context_window is None else context_window,
input_tokens=context_window,
held_output_tokens=default_output_tokens,
output_tokens=default_output_tokens,
)
@ -1709,7 +1721,7 @@ def _estimate_output_tokens(
if _is_input_only_route(route=route):
return 0
requested: Final = _requested_output_tokens(request_body, route)
requested: Final = _requested_output_tokens(request_body)
# Clamp at min(requested-or-default, model_max-or-default). Two purposes:
# (1) Without an explicit cap we still need a finite reservation so the
@ -1739,9 +1751,7 @@ def _gemini_candidate_counts(request_body: Mapping[str, object]) -> tuple[object
)
def _requested_output_tokens(request_body: Mapping[str, object], route: str) -> int | None:
if RouteChecks.is_generate_content_route(route):
return _gemini_requested_output_tokens(request_body)
def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None:
candidates: Final = (
*(request_body.get(field) for field in _OUTPUT_TOKEN_FIELDS),
_nested_field(request_body, "inferenceConfig", "maxTokens"),
@ -1749,7 +1759,7 @@ def _requested_output_tokens(request_body: Mapping[str, object], route: str) ->
return next((tokens for tokens in map(_to_int, candidates) if tokens is not None and tokens >= 0), None)
_GEMINI_PASS_THROUGH_ONLY_CAPS: Final = (
_OTHER_GEMINI_CAP_SPELLINGS: Final = (
("generationConfig", "max_output_tokens"),
("generation_config", "maxOutputTokens"),
("generation_config", "max_output_tokens"),
@ -1758,14 +1768,13 @@ _GEMINI_PASS_THROUGH_ONLY_CAPS: Final = (
def _gemini_requested_output_tokens(request_body: Mapping[str, object]) -> int | None:
gemini_cap: Final = _positive_int(_nested_field(request_body, "generationConfig", "maxOutputTokens"))
pass_through_caps: Final = tuple(
cap
for config_field, field in _GEMINI_PASS_THROUGH_ONLY_CAPS
if (cap := _positive_int(_nested_field(request_body, config_field, field))) is not None
if gemini_cap is None:
return None
other_caps: Final = (
_positive_int(_nested_field(request_body, config_field, field))
for config_field, field in _OTHER_GEMINI_CAP_SPELLINGS
)
if not pass_through_caps:
return gemini_cap
return max(gemini_cap or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, *pass_through_caps)
return max((gemini_cap, *(cap for cap in other_caps if cap is not None)))
def _positive_int(value: object) -> int | None:
@ -1773,46 +1782,10 @@ def _positive_int(value: object) -> int | None:
return number if number is not None and number > 0 else None
def _requested_output_counts(
request_body: Mapping[str, object],
route: str,
model: str,
llm_router: Router | None,
) -> tuple[object, ...]:
if not RouteChecks.is_generate_content_route(route):
return (request_body.get("n"), request_body.get("best_of"))
candidate_counts: Final = _gemini_candidate_counts(request_body)
if all(count is None for count in candidate_counts):
return ()
return candidate_counts if _takes_candidate_count(model=model, llm_router=llm_router) else ()
def _takes_candidate_count(model: str, llm_router: Router | None) -> bool:
deployments: Final = (llm_router.registry().deployments(model) if llm_router is not None else None) or ()
deployment_params: Final = tuple(
params for deployment in deployments if (params := as_str_mapping(_get_value(deployment, "litellm_params")))
)
return any(_serves_generate_content_natively(params) for params in deployment_params or ({"model": model},))
def _serves_generate_content_natively(litellm_params: Mapping[str, object]) -> bool:
model: Final = litellm_params.get("model")
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
if not isinstance(model, str):
return True
try:
backend_model, provider, _, _ = litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
)
native_config: Final = ProviderConfigManager.get_provider_google_genai_generate_content_config(
model=backend_model,
provider=litellm.LlmProviders(provider),
litellm_params={key: litellm_params[key] for key in ("base_model", "api_base") if key in litellm_params},
)
except (litellm.BadRequestError, ValueError):
return True
return native_config is not None
def _requested_output_counts(request_body: Mapping[str, object], route: str) -> tuple[object, ...]:
if RouteChecks.is_generate_content_route(route):
return _gemini_candidate_counts(request_body)
return (request_body.get("n"), request_body.get("best_of"))
def _get_output_multiplier(requested_counts: tuple[object, ...]) -> int:

View file

@ -1,6 +1,46 @@
import json
import traceback
from unittest import mock
from unittest.mock import MagicMock, patch
import httpx
import pytest
from fastapi import Request, Response
from fastapi.testclient import TestClient
from litellm.passthrough.utils import CommonUtils
from unittest.mock import Mock
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
@pytest.mark.asyncio
async def test_get_litellm_virtual_key():
"""
Test that the get_litellm_virtual_key function correctly handles the API key authentication
"""
# Test with x-litellm-api-key
mock_request = Mock()
mock_request.headers = {"x-litellm-api-key": "test-key-123"}
result = get_litellm_virtual_key(mock_request)
assert result == "Bearer test-key-123"
# Test with Authorization header
mock_request.headers = {"Authorization": "Bearer auth-key-456"}
result = get_litellm_virtual_key(mock_request)
assert result == "Bearer auth-key-456"
# Test with both headers (x-litellm-api-key should take precedence)
mock_request.headers = {
"x-litellm-api-key": "test-key-123",
"Authorization": "Bearer auth-key-456",
}
result = get_litellm_virtual_key(mock_request)
assert result == "Bearer test-key-123"
def test_encode_bedrock_runtime_modelid_arn():
# Test application-inference-profile ARN
endpoint = "model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse"

View file

@ -25,7 +25,6 @@ from litellm.proxy.common_utils.user_api_key_cache import (
team_membership_reservation_cache_key,
)
from litellm.proxy.spend_tracking.budget_reservation import (
DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK,
_get_team_member_budget_counter,
estimate_request_input_cost,
estimate_request_max_cost,
@ -197,9 +196,11 @@ GEMINI_SYSTEM_PARTS: Final = ("Answer tersely", "Use British spelling")
(
({"generationConfig": {"maxOutputTokens": 5}}, {"max_tokens": 5}),
({"generationConfig": {"maxOutputTokens": 5, "candidateCount": 3}}, {"max_tokens": 5, "n": 3}),
({"generation_config": {"max_output_tokens": 5, "candidate_count": 3}}, {"n": 3}),
({"generationConfig": {"candidate_count": 3}}, {"n": 3}),
({}, {}),
({"generationConfig": {"maxOutputTokens": 5, "candidate_count": 3}}, {"max_tokens": 5, "n": 3}),
(
{"generationConfig": {"maxOutputTokens": 5}, "generation_config": {"candidate_count": 3}},
{"max_tokens": 5, "n": 3},
),
),
)
@pytest.mark.parametrize("system_texts", (GEMINI_SYSTEM_PARTS[:1], GEMINI_SYSTEM_PARTS))
@ -239,6 +240,7 @@ def test_gemini_generate_content_body_reserves_the_prompt_not_the_context_window
GEMINI_ROUTE: Final = "/v1beta/models/gemini-3.8-flash:generateContent"
GEMINI_TEXT_CONTENT: Final = {"role": "user", "parts": [{"text": GEMINI_PROMPT}]}
GEMINI_AUDIO_PART: Final = {"inlineData": {"mimeType": "audio/wav", "data": "UklGRg" * 40_000}}
GEMINI_OUTPUT_CAP: Final = {"generationConfig": {"maxOutputTokens": 5}}
def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> float | None:
@ -277,7 +279,9 @@ def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> fl
def test_gemini_body_the_prompt_count_cannot_see_whole_keeps_the_context_window_hold(body: dict[str, object]) -> None:
context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"]
assert context_window is not None
assert _gemini_hold(body) == _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window)
assert _gemini_hold({**body, **GEMINI_OUTPUT_CAP}) == _gemini_hold(
{"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}, input_tokens=context_window
)
def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None:
@ -290,10 +294,12 @@ def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None:
{"role": "model", "parts": [{"functionCall": {"name": "lookup", "args": {}}, "thoughtSignature": "c2ln"}]},
{"role": "user", "parts": [{"functionResponse": {"name": "lookup", "response": {"answer": "hi"}}}]},
],
**GEMINI_OUTPUT_CAP,
}
text_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
text_body: Final = {"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}
text_hold: Final = _gemini_hold(text_body)
tool_turns_hold: Final = _gemini_hold(tool_turns)
context_window_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window)
context_window_hold: Final = _gemini_hold(text_body, input_tokens=context_window)
assert text_hold is not None and tool_turns_hold is not None and context_window_hold is not None
assert text_hold < tool_turns_hold < context_window_hold
@ -328,22 +334,34 @@ def test_gemini_output_fields_on_a_chat_body_leave_its_hold_alone(gemini_output_
assert _chat_hold({**GEMINI_CHAT_BODY, **gemini_output_fields}) == plain_hold
@pytest.mark.parametrize("cap", (0, -1000))
def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int) -> None:
uncapped_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
assert uncapped_hold is not None
@pytest.mark.parametrize(
"generation_config",
({}, {"candidateCount": 4}, {"maxOutputTokens": 0}, {"maxOutputTokens": -1000}, {"maxOutputTokens": "many"}),
)
def test_a_gemini_call_without_an_output_cap_holds_the_whole_context_window(
generation_config: dict[str, object],
) -> None:
context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"]
assert context_window is not None
window_filling_chat_hold: Final = estimate_request_max_cost(
request_body=GEMINI_CHAT_BODY,
route="/chat/completions",
llm_router=None,
input_token_counts={"gemini-3.8-flash": context_window},
)
assert window_filling_chat_hold is not None
assert (
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": cap}})
== uncapped_hold
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": generation_config})
== window_filling_chat_hold
)
@pytest.mark.parametrize(
("output_fields", "same_hold_fields"),
(
({"generationConfig": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
({"generation_config": {"maxOutputTokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
({"generation_config": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
({"generationConfig": {"max_output_tokens": 30000}}, {}),
({"generation_config": {"maxOutputTokens": 30000}}, {}),
({"generation_config": {"max_output_tokens": 30000}}, {}),
(
{"generationConfig": {"maxOutputTokens": 8000}, "generation_config": {"max_output_tokens": 30000}},
{"generationConfig": {"maxOutputTokens": 30000}},
@ -355,7 +373,7 @@ def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int)
),
),
)
def test_an_output_cap_spelling_v1beta_drops_can_only_raise_the_hold(
def test_an_output_cap_spelled_other_than_max_output_tokens_can_only_raise_the_hold(
output_fields: dict[str, object], same_hold_fields: dict[str, object]
) -> None:
expected_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **same_hold_fields})
@ -368,10 +386,12 @@ UNPRICEABLE_COUNTS: Final = (10**400, "9" * 400, 1e308, math.inf)
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
def test_a_candidate_count_too_large_to_price_holds_one_candidate(count: object) -> None:
one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP})
assert one_candidate_hold is not None
assert (
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"candidateCount": count}})
_gemini_hold(
{"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": 5, "candidateCount": count}}
)
== one_candidate_hold
)
@ -412,8 +432,8 @@ def _priced_gemini_hold(
(
(WIDE_WINDOW, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}),
(WIDE_WINDOW, {"maxOutputTokens": 20000, "candidate_count": 2}, WIDE_WINDOW, {"maxOutputTokens": 40000}),
(WIDE_WINDOW, {"candidateCount": 2}, WIDE_WINDOW, {"maxOutputTokens": 2 * DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK}),
(LOW_OUTPUT_CEILING, {"candidateCount": 4}, {**WIDE_WINDOW, "max_output_tokens": 4000}, {}),
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}),
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000}, WIDE_WINDOW, {"maxOutputTokens": 8000}),
),
)
def test_candidates_hold_what_one_candidate_writing_all_their_output_holds(
@ -433,9 +453,10 @@ def test_candidates_hold_what_one_candidate_writing_all_their_output_holds(
(WIDE_WINDOW, {"candidateCount": 1e300}),
(WIDE_WINDOW, {"maxOutputTokens": 100_000, "candidateCount": 8}),
(LOW_OUTPUT_CEILING, {"candidateCount": 1e300}),
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 100_000, "candidateCount": 8}),
),
)
def test_candidates_never_hold_more_than_the_whole_context_window_hold(
def test_a_gemini_hold_never_exceeds_the_whole_context_window_hold(
model_info: dict[str, object], generation_config: dict[str, object]
) -> None:
context_window_hold: Final = _priced_gemini_hold(model_info, {}, input_tokens=1_000_000)
@ -443,31 +464,14 @@ def test_candidates_never_hold_more_than_the_whole_context_window_hold(
assert _priced_gemini_hold(model_info, generation_config) == context_window_hold
def test_candidates_never_hold_less_than_one_candidate() -> None:
narrow_window: Final = {"max_input_tokens": 10, "max_output_tokens": 100_000}
one_candidate_hold: Final = _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000})
assert one_candidate_hold is not None
assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold
NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry"
@pytest.mark.parametrize(
("generation_config", "same_hold_config"),
(
({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}),
({"maxOutputTokens": 8000, "candidateCount": 4}, {}),
({"candidateCount": 1_000_000}, {}),
),
"generation_config", ({}, {"maxOutputTokens": 4000}, {"maxOutputTokens": 8000, "candidateCount": 4})
)
def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate(
generation_config: dict[str, object], same_hold_config: dict[str, object]
) -> None:
no_window: Final = {"max_output_tokens": 100_000}
expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL)
assert expected_hold is not None
assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold
def test_a_gemini_model_with_no_context_window_is_not_held(generation_config: dict[str, object]) -> None:
assert _priced_gemini_hold({"max_output_tokens": 100_000}, generation_config, model=NO_WINDOW_MODEL) is None
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
@ -487,18 +491,17 @@ def _group_generate_content_hold(router: Router, generation_config: dict[str, ob
@pytest.mark.parametrize(
("backend_models", "candidates_are_generated"),
"backend_models",
(
(("vertex_ai/claude-haiku-5-5",), False),
(("anthropic/claude-haiku-5-5",), False),
(("gemini/gemini-3.8-flash",), True),
(("vertex_ai/gemini-3.8-flash",), True),
(("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"), True),
("vertex_ai/claude-haiku-5-5",),
("anthropic/claude-haiku-5-5",),
("openai/gpt-5.4-mini",),
("gemini/gemini-3.8-flash",),
("vertex_ai/gemini-3.8-flash",),
("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"),
),
)
def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidates(
backend_models: tuple[str, ...], candidates_are_generated: bool
) -> None:
def test_a_candidate_count_is_held_whatever_deployment_serves_the_name(backend_models: tuple[str, ...]) -> None:
router: Final = Router(
model_list=[
{"model_name": "assistant", "litellm_params": {"model": model, "api_key": "sk-test"}}
@ -510,8 +513,7 @@ def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidat
assert one_candidate_hold is not None
assert three_candidate_hold is not None
assert (three_candidate_hold > one_candidate_hold) is candidates_are_generated
assert three_candidate_hold >= one_candidate_hold
assert three_candidate_hold > one_candidate_hold
def test_a_candidate_count_for_a_gemini_name_no_deployment_serves_is_held() -> None: