mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep main's Gemini hold unless the request caps its output
An uncapped generateContent call, or one capped only by a snake_case spelling, now holds what main holds (the context window plus the default output). A call with generationConfig.maxOutputTokens holds its counted prompt plus cap times candidateCount, never clamped to the deployment's max_output_tokens and never more than main's hold. A model with no context window keeps main's no-hold, and candidateCount always counts on generateContent routes. Also keeps get_litellm_virtual_key in pass_through_endpoints/common_utils.py, so an out-of-tree importer does not break
This commit is contained in:
parent
7df5e93f68
commit
82ac5a1ef9
4 changed files with 156 additions and 123 deletions
18
litellm/proxy/pass_through_endpoints/common_utils.py
Normal file
18
litellm/proxy/pass_through_endpoints/common_utils.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from typing import Final
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
|
||||
def get_litellm_virtual_key(request: Request) -> str:
|
||||
"""
|
||||
Extract and format API key from request headers.
|
||||
Prioritizes x-litellm-api-key over Authorization header.
|
||||
|
||||
|
||||
Vertex JS SDK uses `Authorization` header, we use `x-litellm-api-key` to pass litellm virtual key
|
||||
|
||||
"""
|
||||
litellm_api_key: Final = request.headers.get("x-litellm-api-key")
|
||||
if litellm_api_key:
|
||||
return f"Bearer {litellm_api_key}"
|
||||
return request.headers.get("Authorization", "")
|
||||
|
|
@ -55,7 +55,6 @@ from litellm.router import Router
|
|||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1404,9 +1403,7 @@ def _estimate_request_max_cost_for_model(
|
|||
llm_router: Router | None,
|
||||
input_tokens: int | None = None,
|
||||
) -> float | None:
|
||||
output_multiplier: Final = _get_output_multiplier(
|
||||
_requested_output_counts(request_body=request_body, route=route, model=model, llm_router=llm_router)
|
||||
)
|
||||
output_multiplier: Final = _get_output_multiplier(_requested_output_counts(request_body=request_body, route=route))
|
||||
estimates: Final = [
|
||||
_max_cost_for_cost_info(
|
||||
request_body=request_body,
|
||||
|
|
@ -1444,39 +1441,54 @@ def _max_cost_for_cost_info(
|
|||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
)
|
||||
if estimated_input_tokens is None:
|
||||
return None
|
||||
if RouteChecks.is_generate_content_route(route):
|
||||
return _generate_content_hold(
|
||||
request_body=request_body,
|
||||
model_info=model_info,
|
||||
input_tokens=estimated_input_tokens,
|
||||
output_multiplier=output_multiplier,
|
||||
)
|
||||
output_tokens: Final = _estimate_output_tokens(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model_info=model_info,
|
||||
)
|
||||
if estimated_input_tokens is None or output_tokens is None:
|
||||
if output_tokens is None:
|
||||
return None
|
||||
candidates_hold: Final = _priced_hold(
|
||||
return _priced_hold(
|
||||
model_info=model_info,
|
||||
input_tokens=estimated_input_tokens,
|
||||
held_output_tokens=output_tokens * output_multiplier,
|
||||
output_tokens=output_tokens,
|
||||
)
|
||||
if candidates_hold is None or output_multiplier == 1 or not RouteChecks.is_generate_content_route(route):
|
||||
return candidates_hold
|
||||
one_candidate_hold: Final = _priced_hold(
|
||||
|
||||
|
||||
def _generate_content_hold(
|
||||
request_body: Mapping[str, object], model_info: Mapping[str, object], input_tokens: int, output_multiplier: int
|
||||
) -> float | None:
|
||||
main_hold: Final = _main_generate_content_hold(model_info)
|
||||
requested_output_tokens: Final = _gemini_requested_output_tokens(request_body)
|
||||
if main_hold is None or requested_output_tokens is None:
|
||||
return main_hold
|
||||
capped_hold: Final = _priced_hold(
|
||||
model_info=model_info,
|
||||
input_tokens=estimated_input_tokens,
|
||||
held_output_tokens=output_tokens,
|
||||
output_tokens=output_tokens,
|
||||
input_tokens=input_tokens,
|
||||
held_output_tokens=requested_output_tokens * output_multiplier,
|
||||
output_tokens=requested_output_tokens,
|
||||
)
|
||||
hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens)
|
||||
if one_candidate_hold is None or hold_ceiling is None:
|
||||
return one_candidate_hold
|
||||
return max(one_candidate_hold, min(candidates_hold, hold_ceiling))
|
||||
return main_hold if capped_hold is None else min(capped_hold, main_hold)
|
||||
|
||||
|
||||
def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None:
|
||||
def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None:
|
||||
context_window: Final = _to_int(model_info.get("max_input_tokens"))
|
||||
if context_window is None:
|
||||
return None
|
||||
default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info))
|
||||
return _priced_hold(
|
||||
model_info=model_info,
|
||||
input_tokens=counted_input_tokens if context_window is None else context_window,
|
||||
input_tokens=context_window,
|
||||
held_output_tokens=default_output_tokens,
|
||||
output_tokens=default_output_tokens,
|
||||
)
|
||||
|
|
@ -1709,7 +1721,7 @@ def _estimate_output_tokens(
|
|||
if _is_input_only_route(route=route):
|
||||
return 0
|
||||
|
||||
requested: Final = _requested_output_tokens(request_body, route)
|
||||
requested: Final = _requested_output_tokens(request_body)
|
||||
|
||||
# Clamp at min(requested-or-default, model_max-or-default). Two purposes:
|
||||
# (1) Without an explicit cap we still need a finite reservation so the
|
||||
|
|
@ -1739,9 +1751,7 @@ def _gemini_candidate_counts(request_body: Mapping[str, object]) -> tuple[object
|
|||
)
|
||||
|
||||
|
||||
def _requested_output_tokens(request_body: Mapping[str, object], route: str) -> int | None:
|
||||
if RouteChecks.is_generate_content_route(route):
|
||||
return _gemini_requested_output_tokens(request_body)
|
||||
def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None:
|
||||
candidates: Final = (
|
||||
*(request_body.get(field) for field in _OUTPUT_TOKEN_FIELDS),
|
||||
_nested_field(request_body, "inferenceConfig", "maxTokens"),
|
||||
|
|
@ -1749,7 +1759,7 @@ def _requested_output_tokens(request_body: Mapping[str, object], route: str) ->
|
|||
return next((tokens for tokens in map(_to_int, candidates) if tokens is not None and tokens >= 0), None)
|
||||
|
||||
|
||||
_GEMINI_PASS_THROUGH_ONLY_CAPS: Final = (
|
||||
_OTHER_GEMINI_CAP_SPELLINGS: Final = (
|
||||
("generationConfig", "max_output_tokens"),
|
||||
("generation_config", "maxOutputTokens"),
|
||||
("generation_config", "max_output_tokens"),
|
||||
|
|
@ -1758,14 +1768,13 @@ _GEMINI_PASS_THROUGH_ONLY_CAPS: Final = (
|
|||
|
||||
def _gemini_requested_output_tokens(request_body: Mapping[str, object]) -> int | None:
|
||||
gemini_cap: Final = _positive_int(_nested_field(request_body, "generationConfig", "maxOutputTokens"))
|
||||
pass_through_caps: Final = tuple(
|
||||
cap
|
||||
for config_field, field in _GEMINI_PASS_THROUGH_ONLY_CAPS
|
||||
if (cap := _positive_int(_nested_field(request_body, config_field, field))) is not None
|
||||
if gemini_cap is None:
|
||||
return None
|
||||
other_caps: Final = (
|
||||
_positive_int(_nested_field(request_body, config_field, field))
|
||||
for config_field, field in _OTHER_GEMINI_CAP_SPELLINGS
|
||||
)
|
||||
if not pass_through_caps:
|
||||
return gemini_cap
|
||||
return max(gemini_cap or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, *pass_through_caps)
|
||||
return max((gemini_cap, *(cap for cap in other_caps if cap is not None)))
|
||||
|
||||
|
||||
def _positive_int(value: object) -> int | None:
|
||||
|
|
@ -1773,46 +1782,10 @@ def _positive_int(value: object) -> int | None:
|
|||
return number if number is not None and number > 0 else None
|
||||
|
||||
|
||||
def _requested_output_counts(
|
||||
request_body: Mapping[str, object],
|
||||
route: str,
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
) -> tuple[object, ...]:
|
||||
if not RouteChecks.is_generate_content_route(route):
|
||||
return (request_body.get("n"), request_body.get("best_of"))
|
||||
candidate_counts: Final = _gemini_candidate_counts(request_body)
|
||||
if all(count is None for count in candidate_counts):
|
||||
return ()
|
||||
return candidate_counts if _takes_candidate_count(model=model, llm_router=llm_router) else ()
|
||||
|
||||
|
||||
def _takes_candidate_count(model: str, llm_router: Router | None) -> bool:
|
||||
deployments: Final = (llm_router.registry().deployments(model) if llm_router is not None else None) or ()
|
||||
deployment_params: Final = tuple(
|
||||
params for deployment in deployments if (params := as_str_mapping(_get_value(deployment, "litellm_params")))
|
||||
)
|
||||
return any(_serves_generate_content_natively(params) for params in deployment_params or ({"model": model},))
|
||||
|
||||
|
||||
def _serves_generate_content_natively(litellm_params: Mapping[str, object]) -> bool:
|
||||
model: Final = litellm_params.get("model")
|
||||
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
|
||||
if not isinstance(model, str):
|
||||
return True
|
||||
try:
|
||||
backend_model, provider, _, _ = litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
)
|
||||
native_config: Final = ProviderConfigManager.get_provider_google_genai_generate_content_config(
|
||||
model=backend_model,
|
||||
provider=litellm.LlmProviders(provider),
|
||||
litellm_params={key: litellm_params[key] for key in ("base_model", "api_base") if key in litellm_params},
|
||||
)
|
||||
except (litellm.BadRequestError, ValueError):
|
||||
return True
|
||||
return native_config is not None
|
||||
def _requested_output_counts(request_body: Mapping[str, object], route: str) -> tuple[object, ...]:
|
||||
if RouteChecks.is_generate_content_route(route):
|
||||
return _gemini_candidate_counts(request_body)
|
||||
return (request_body.get("n"), request_body.get("best_of"))
|
||||
|
||||
|
||||
def _get_output_multiplier(requested_counts: tuple[object, ...]) -> int:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,46 @@
|
|||
import json
|
||||
import traceback
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import Request, Response
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.passthrough.utils import CommonUtils
|
||||
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_litellm_virtual_key():
|
||||
"""
|
||||
Test that the get_litellm_virtual_key function correctly handles the API key authentication
|
||||
"""
|
||||
# Test with x-litellm-api-key
|
||||
mock_request = Mock()
|
||||
mock_request.headers = {"x-litellm-api-key": "test-key-123"}
|
||||
result = get_litellm_virtual_key(mock_request)
|
||||
assert result == "Bearer test-key-123"
|
||||
|
||||
# Test with Authorization header
|
||||
mock_request.headers = {"Authorization": "Bearer auth-key-456"}
|
||||
result = get_litellm_virtual_key(mock_request)
|
||||
assert result == "Bearer auth-key-456"
|
||||
|
||||
# Test with both headers (x-litellm-api-key should take precedence)
|
||||
mock_request.headers = {
|
||||
"x-litellm-api-key": "test-key-123",
|
||||
"Authorization": "Bearer auth-key-456",
|
||||
}
|
||||
result = get_litellm_virtual_key(mock_request)
|
||||
assert result == "Bearer test-key-123"
|
||||
|
||||
|
||||
def test_encode_bedrock_runtime_modelid_arn():
|
||||
# Test application-inference-profile ARN
|
||||
endpoint = "model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse"
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK,
|
||||
_get_team_member_budget_counter,
|
||||
estimate_request_input_cost,
|
||||
estimate_request_max_cost,
|
||||
|
|
@ -197,9 +196,11 @@ GEMINI_SYSTEM_PARTS: Final = ("Answer tersely", "Use British spelling")
|
|||
(
|
||||
({"generationConfig": {"maxOutputTokens": 5}}, {"max_tokens": 5}),
|
||||
({"generationConfig": {"maxOutputTokens": 5, "candidateCount": 3}}, {"max_tokens": 5, "n": 3}),
|
||||
({"generation_config": {"max_output_tokens": 5, "candidate_count": 3}}, {"n": 3}),
|
||||
({"generationConfig": {"candidate_count": 3}}, {"n": 3}),
|
||||
({}, {}),
|
||||
({"generationConfig": {"maxOutputTokens": 5, "candidate_count": 3}}, {"max_tokens": 5, "n": 3}),
|
||||
(
|
||||
{"generationConfig": {"maxOutputTokens": 5}, "generation_config": {"candidate_count": 3}},
|
||||
{"max_tokens": 5, "n": 3},
|
||||
),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("system_texts", (GEMINI_SYSTEM_PARTS[:1], GEMINI_SYSTEM_PARTS))
|
||||
|
|
@ -239,6 +240,7 @@ def test_gemini_generate_content_body_reserves_the_prompt_not_the_context_window
|
|||
GEMINI_ROUTE: Final = "/v1beta/models/gemini-3.8-flash:generateContent"
|
||||
GEMINI_TEXT_CONTENT: Final = {"role": "user", "parts": [{"text": GEMINI_PROMPT}]}
|
||||
GEMINI_AUDIO_PART: Final = {"inlineData": {"mimeType": "audio/wav", "data": "UklGRg" * 40_000}}
|
||||
GEMINI_OUTPUT_CAP: Final = {"generationConfig": {"maxOutputTokens": 5}}
|
||||
|
||||
|
||||
def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> float | None:
|
||||
|
|
@ -277,7 +279,9 @@ def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> fl
|
|||
def test_gemini_body_the_prompt_count_cannot_see_whole_keeps_the_context_window_hold(body: dict[str, object]) -> None:
|
||||
context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"]
|
||||
assert context_window is not None
|
||||
assert _gemini_hold(body) == _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window)
|
||||
assert _gemini_hold({**body, **GEMINI_OUTPUT_CAP}) == _gemini_hold(
|
||||
{"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}, input_tokens=context_window
|
||||
)
|
||||
|
||||
|
||||
def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None:
|
||||
|
|
@ -290,10 +294,12 @@ def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None:
|
|||
{"role": "model", "parts": [{"functionCall": {"name": "lookup", "args": {}}, "thoughtSignature": "c2ln"}]},
|
||||
{"role": "user", "parts": [{"functionResponse": {"name": "lookup", "response": {"answer": "hi"}}}]},
|
||||
],
|
||||
**GEMINI_OUTPUT_CAP,
|
||||
}
|
||||
text_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
|
||||
text_body: Final = {"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}
|
||||
text_hold: Final = _gemini_hold(text_body)
|
||||
tool_turns_hold: Final = _gemini_hold(tool_turns)
|
||||
context_window_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window)
|
||||
context_window_hold: Final = _gemini_hold(text_body, input_tokens=context_window)
|
||||
assert text_hold is not None and tool_turns_hold is not None and context_window_hold is not None
|
||||
assert text_hold < tool_turns_hold < context_window_hold
|
||||
|
||||
|
|
@ -328,22 +334,34 @@ def test_gemini_output_fields_on_a_chat_body_leave_its_hold_alone(gemini_output_
|
|||
assert _chat_hold({**GEMINI_CHAT_BODY, **gemini_output_fields}) == plain_hold
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cap", (0, -1000))
|
||||
def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int) -> None:
|
||||
uncapped_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
|
||||
assert uncapped_hold is not None
|
||||
@pytest.mark.parametrize(
|
||||
"generation_config",
|
||||
({}, {"candidateCount": 4}, {"maxOutputTokens": 0}, {"maxOutputTokens": -1000}, {"maxOutputTokens": "many"}),
|
||||
)
|
||||
def test_a_gemini_call_without_an_output_cap_holds_the_whole_context_window(
|
||||
generation_config: dict[str, object],
|
||||
) -> None:
|
||||
context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"]
|
||||
assert context_window is not None
|
||||
window_filling_chat_hold: Final = estimate_request_max_cost(
|
||||
request_body=GEMINI_CHAT_BODY,
|
||||
route="/chat/completions",
|
||||
llm_router=None,
|
||||
input_token_counts={"gemini-3.8-flash": context_window},
|
||||
)
|
||||
assert window_filling_chat_hold is not None
|
||||
assert (
|
||||
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": cap}})
|
||||
== uncapped_hold
|
||||
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": generation_config})
|
||||
== window_filling_chat_hold
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_fields", "same_hold_fields"),
|
||||
(
|
||||
({"generationConfig": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
|
||||
({"generation_config": {"maxOutputTokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
|
||||
({"generation_config": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}),
|
||||
({"generationConfig": {"max_output_tokens": 30000}}, {}),
|
||||
({"generation_config": {"maxOutputTokens": 30000}}, {}),
|
||||
({"generation_config": {"max_output_tokens": 30000}}, {}),
|
||||
(
|
||||
{"generationConfig": {"maxOutputTokens": 8000}, "generation_config": {"max_output_tokens": 30000}},
|
||||
{"generationConfig": {"maxOutputTokens": 30000}},
|
||||
|
|
@ -355,7 +373,7 @@ def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int)
|
|||
),
|
||||
),
|
||||
)
|
||||
def test_an_output_cap_spelling_v1beta_drops_can_only_raise_the_hold(
|
||||
def test_an_output_cap_spelled_other_than_max_output_tokens_can_only_raise_the_hold(
|
||||
output_fields: dict[str, object], same_hold_fields: dict[str, object]
|
||||
) -> None:
|
||||
expected_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **same_hold_fields})
|
||||
|
|
@ -368,10 +386,12 @@ UNPRICEABLE_COUNTS: Final = (10**400, "9" * 400, 1e308, math.inf)
|
|||
|
||||
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
|
||||
def test_a_candidate_count_too_large_to_price_holds_one_candidate(count: object) -> None:
|
||||
one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]})
|
||||
one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP})
|
||||
assert one_candidate_hold is not None
|
||||
assert (
|
||||
_gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"candidateCount": count}})
|
||||
_gemini_hold(
|
||||
{"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": 5, "candidateCount": count}}
|
||||
)
|
||||
== one_candidate_hold
|
||||
)
|
||||
|
||||
|
|
@ -412,8 +432,8 @@ def _priced_gemini_hold(
|
|||
(
|
||||
(WIDE_WINDOW, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}),
|
||||
(WIDE_WINDOW, {"maxOutputTokens": 20000, "candidate_count": 2}, WIDE_WINDOW, {"maxOutputTokens": 40000}),
|
||||
(WIDE_WINDOW, {"candidateCount": 2}, WIDE_WINDOW, {"maxOutputTokens": 2 * DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK}),
|
||||
(LOW_OUTPUT_CEILING, {"candidateCount": 4}, {**WIDE_WINDOW, "max_output_tokens": 4000}, {}),
|
||||
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}),
|
||||
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000}, WIDE_WINDOW, {"maxOutputTokens": 8000}),
|
||||
),
|
||||
)
|
||||
def test_candidates_hold_what_one_candidate_writing_all_their_output_holds(
|
||||
|
|
@ -433,9 +453,10 @@ def test_candidates_hold_what_one_candidate_writing_all_their_output_holds(
|
|||
(WIDE_WINDOW, {"candidateCount": 1e300}),
|
||||
(WIDE_WINDOW, {"maxOutputTokens": 100_000, "candidateCount": 8}),
|
||||
(LOW_OUTPUT_CEILING, {"candidateCount": 1e300}),
|
||||
(LOW_OUTPUT_CEILING, {"maxOutputTokens": 100_000, "candidateCount": 8}),
|
||||
),
|
||||
)
|
||||
def test_candidates_never_hold_more_than_the_whole_context_window_hold(
|
||||
def test_a_gemini_hold_never_exceeds_the_whole_context_window_hold(
|
||||
model_info: dict[str, object], generation_config: dict[str, object]
|
||||
) -> None:
|
||||
context_window_hold: Final = _priced_gemini_hold(model_info, {}, input_tokens=1_000_000)
|
||||
|
|
@ -443,31 +464,14 @@ def test_candidates_never_hold_more_than_the_whole_context_window_hold(
|
|||
assert _priced_gemini_hold(model_info, generation_config) == context_window_hold
|
||||
|
||||
|
||||
def test_candidates_never_hold_less_than_one_candidate() -> None:
|
||||
narrow_window: Final = {"max_input_tokens": 10, "max_output_tokens": 100_000}
|
||||
one_candidate_hold: Final = _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000})
|
||||
assert one_candidate_hold is not None
|
||||
assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold
|
||||
|
||||
|
||||
NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("generation_config", "same_hold_config"),
|
||||
(
|
||||
({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}),
|
||||
({"maxOutputTokens": 8000, "candidateCount": 4}, {}),
|
||||
({"candidateCount": 1_000_000}, {}),
|
||||
),
|
||||
"generation_config", ({}, {"maxOutputTokens": 4000}, {"maxOutputTokens": 8000, "candidateCount": 4})
|
||||
)
|
||||
def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate(
|
||||
generation_config: dict[str, object], same_hold_config: dict[str, object]
|
||||
) -> None:
|
||||
no_window: Final = {"max_output_tokens": 100_000}
|
||||
expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL)
|
||||
assert expected_hold is not None
|
||||
assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold
|
||||
def test_a_gemini_model_with_no_context_window_is_not_held(generation_config: dict[str, object]) -> None:
|
||||
assert _priced_gemini_hold({"max_output_tokens": 100_000}, generation_config, model=NO_WINDOW_MODEL) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("count", UNPRICEABLE_COUNTS)
|
||||
|
|
@ -487,18 +491,17 @@ def _group_generate_content_hold(router: Router, generation_config: dict[str, ob
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_models", "candidates_are_generated"),
|
||||
"backend_models",
|
||||
(
|
||||
(("vertex_ai/claude-haiku-5-5",), False),
|
||||
(("anthropic/claude-haiku-5-5",), False),
|
||||
(("gemini/gemini-3.8-flash",), True),
|
||||
(("vertex_ai/gemini-3.8-flash",), True),
|
||||
(("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"), True),
|
||||
("vertex_ai/claude-haiku-5-5",),
|
||||
("anthropic/claude-haiku-5-5",),
|
||||
("openai/gpt-5.4-mini",),
|
||||
("gemini/gemini-3.8-flash",),
|
||||
("vertex_ai/gemini-3.8-flash",),
|
||||
("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"),
|
||||
),
|
||||
)
|
||||
def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidates(
|
||||
backend_models: tuple[str, ...], candidates_are_generated: bool
|
||||
) -> None:
|
||||
def test_a_candidate_count_is_held_whatever_deployment_serves_the_name(backend_models: tuple[str, ...]) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "assistant", "litellm_params": {"model": model, "api_key": "sk-test"}}
|
||||
|
|
@ -510,8 +513,7 @@ def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidat
|
|||
|
||||
assert one_candidate_hold is not None
|
||||
assert three_candidate_hold is not None
|
||||
assert (three_candidate_hold > one_candidate_hold) is candidates_are_generated
|
||||
assert three_candidate_hold >= one_candidate_hold
|
||||
assert three_candidate_hold > one_candidate_hold
|
||||
|
||||
|
||||
def test_a_candidate_count_for_a_gemini_name_no_deployment_serves_is_held() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue