diff --git a/litellm/proxy/pass_through_endpoints/common_utils.py b/litellm/proxy/pass_through_endpoints/common_utils.py new file mode 100644 index 00000000000..3ae83cbe906 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/common_utils.py @@ -0,0 +1,18 @@ +from typing import Final + +from fastapi import Request + + +def get_litellm_virtual_key(request: Request) -> str: + """ + Extract and format API key from request headers. + Prioritizes x-litellm-api-key over Authorization header. + + + Vertex JS SDK uses `Authorization` header, we use `x-litellm-api-key` to pass litellm virtual key + + """ + litellm_api_key: Final = request.headers.get("x-litellm-api-key") + if litellm_api_key: + return f"Bearer {litellm_api_key}" + return request.headers.get("Authorization", "") diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index a2e0337ce02..173ba3f637c 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -55,7 +55,6 @@ from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict from litellm.types.utils import CallTypes -from litellm.utils import ProviderConfigManager @dataclass @@ -1404,9 +1403,7 @@ def _estimate_request_max_cost_for_model( llm_router: Router | None, input_tokens: int | None = None, ) -> float | None: - output_multiplier: Final = _get_output_multiplier( - _requested_output_counts(request_body=request_body, route=route, model=model, llm_router=llm_router) - ) + output_multiplier: Final = _get_output_multiplier(_requested_output_counts(request_body=request_body, route=route)) estimates: Final = [ _max_cost_for_cost_info( request_body=request_body, @@ -1444,39 +1441,54 @@ def _max_cost_for_cost_info( model_info=model_info, input_tokens=input_tokens, ) + if estimated_input_tokens is None: + return None + if RouteChecks.is_generate_content_route(route): + return _generate_content_hold( + request_body=request_body, + model_info=model_info, + input_tokens=estimated_input_tokens, + output_multiplier=output_multiplier, + ) output_tokens: Final = _estimate_output_tokens( request_body=request_body, route=route, model_info=model_info, ) - if estimated_input_tokens is None or output_tokens is None: + if output_tokens is None: return None - candidates_hold: Final = _priced_hold( + return _priced_hold( model_info=model_info, input_tokens=estimated_input_tokens, held_output_tokens=output_tokens * output_multiplier, output_tokens=output_tokens, ) - if candidates_hold is None or output_multiplier == 1 or not RouteChecks.is_generate_content_route(route): - return candidates_hold - one_candidate_hold: Final = _priced_hold( + + +def _generate_content_hold( + request_body: Mapping[str, object], model_info: Mapping[str, object], input_tokens: int, output_multiplier: int +) -> float | None: + main_hold: Final = _main_generate_content_hold(model_info) + requested_output_tokens: Final = _gemini_requested_output_tokens(request_body) + if main_hold is None or requested_output_tokens is None: + return main_hold + capped_hold: Final = _priced_hold( model_info=model_info, - input_tokens=estimated_input_tokens, - held_output_tokens=output_tokens, - output_tokens=output_tokens, + input_tokens=input_tokens, + held_output_tokens=requested_output_tokens * output_multiplier, + output_tokens=requested_output_tokens, ) - hold_ceiling: Final = _candidates_hold_ceiling(model_info=model_info, counted_input_tokens=estimated_input_tokens) - if one_candidate_hold is None or hold_ceiling is None: - return one_candidate_hold - return max(one_candidate_hold, min(candidates_hold, hold_ceiling)) + return main_hold if capped_hold is None else min(capped_hold, main_hold) -def _candidates_hold_ceiling(model_info: Mapping[str, object], counted_input_tokens: int) -> float | None: +def _main_generate_content_hold(model_info: Mapping[str, object]) -> float | None: context_window: Final = _to_int(model_info.get("max_input_tokens")) + if context_window is None: + return None default_output_tokens: Final = min(DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _model_output_ceiling(model_info)) return _priced_hold( model_info=model_info, - input_tokens=counted_input_tokens if context_window is None else context_window, + input_tokens=context_window, held_output_tokens=default_output_tokens, output_tokens=default_output_tokens, ) @@ -1709,7 +1721,7 @@ def _estimate_output_tokens( if _is_input_only_route(route=route): return 0 - requested: Final = _requested_output_tokens(request_body, route) + requested: Final = _requested_output_tokens(request_body) # Clamp at min(requested-or-default, model_max-or-default). Two purposes: # (1) Without an explicit cap we still need a finite reservation so the @@ -1739,9 +1751,7 @@ def _gemini_candidate_counts(request_body: Mapping[str, object]) -> tuple[object ) -def _requested_output_tokens(request_body: Mapping[str, object], route: str) -> int | None: - if RouteChecks.is_generate_content_route(route): - return _gemini_requested_output_tokens(request_body) +def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None: candidates: Final = ( *(request_body.get(field) for field in _OUTPUT_TOKEN_FIELDS), _nested_field(request_body, "inferenceConfig", "maxTokens"), @@ -1749,7 +1759,7 @@ def _requested_output_tokens(request_body: Mapping[str, object], route: str) -> return next((tokens for tokens in map(_to_int, candidates) if tokens is not None and tokens >= 0), None) -_GEMINI_PASS_THROUGH_ONLY_CAPS: Final = ( +_OTHER_GEMINI_CAP_SPELLINGS: Final = ( ("generationConfig", "max_output_tokens"), ("generation_config", "maxOutputTokens"), ("generation_config", "max_output_tokens"), @@ -1758,14 +1768,13 @@ _GEMINI_PASS_THROUGH_ONLY_CAPS: Final = ( def _gemini_requested_output_tokens(request_body: Mapping[str, object]) -> int | None: gemini_cap: Final = _positive_int(_nested_field(request_body, "generationConfig", "maxOutputTokens")) - pass_through_caps: Final = tuple( - cap - for config_field, field in _GEMINI_PASS_THROUGH_ONLY_CAPS - if (cap := _positive_int(_nested_field(request_body, config_field, field))) is not None + if gemini_cap is None: + return None + other_caps: Final = ( + _positive_int(_nested_field(request_body, config_field, field)) + for config_field, field in _OTHER_GEMINI_CAP_SPELLINGS ) - if not pass_through_caps: - return gemini_cap - return max(gemini_cap or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, *pass_through_caps) + return max((gemini_cap, *(cap for cap in other_caps if cap is not None))) def _positive_int(value: object) -> int | None: @@ -1773,46 +1782,10 @@ def _positive_int(value: object) -> int | None: return number if number is not None and number > 0 else None -def _requested_output_counts( - request_body: Mapping[str, object], - route: str, - model: str, - llm_router: Router | None, -) -> tuple[object, ...]: - if not RouteChecks.is_generate_content_route(route): - return (request_body.get("n"), request_body.get("best_of")) - candidate_counts: Final = _gemini_candidate_counts(request_body) - if all(count is None for count in candidate_counts): - return () - return candidate_counts if _takes_candidate_count(model=model, llm_router=llm_router) else () - - -def _takes_candidate_count(model: str, llm_router: Router | None) -> bool: - deployments: Final = (llm_router.registry().deployments(model) if llm_router is not None else None) or () - deployment_params: Final = tuple( - params for deployment in deployments if (params := as_str_mapping(_get_value(deployment, "litellm_params"))) - ) - return any(_serves_generate_content_natively(params) for params in deployment_params or ({"model": model},)) - - -def _serves_generate_content_natively(litellm_params: Mapping[str, object]) -> bool: - model: Final = litellm_params.get("model") - custom_llm_provider: Final = litellm_params.get("custom_llm_provider") - if not isinstance(model, str): - return True - try: - backend_model, provider, _, _ = litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, - ) - native_config: Final = ProviderConfigManager.get_provider_google_genai_generate_content_config( - model=backend_model, - provider=litellm.LlmProviders(provider), - litellm_params={key: litellm_params[key] for key in ("base_model", "api_base") if key in litellm_params}, - ) - except (litellm.BadRequestError, ValueError): - return True - return native_config is not None +def _requested_output_counts(request_body: Mapping[str, object], route: str) -> tuple[object, ...]: + if RouteChecks.is_generate_content_route(route): + return _gemini_candidate_counts(request_body) + return (request_body.get("n"), request_body.get("best_of")) def _get_output_multiplier(requested_counts: tuple[object, ...]) -> int: diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py index 54161b4883b..078bd4dd402 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py @@ -1,6 +1,46 @@ +import json +import traceback +from unittest import mock +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from fastapi import Request, Response +from fastapi.testclient import TestClient + from litellm.passthrough.utils import CommonUtils +from unittest.mock import Mock + +from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key + + +@pytest.mark.asyncio +async def test_get_litellm_virtual_key(): + """ + Test that the get_litellm_virtual_key function correctly handles the API key authentication + """ + # Test with x-litellm-api-key + mock_request = Mock() + mock_request.headers = {"x-litellm-api-key": "test-key-123"} + result = get_litellm_virtual_key(mock_request) + assert result == "Bearer test-key-123" + + # Test with Authorization header + mock_request.headers = {"Authorization": "Bearer auth-key-456"} + result = get_litellm_virtual_key(mock_request) + assert result == "Bearer auth-key-456" + + # Test with both headers (x-litellm-api-key should take precedence) + mock_request.headers = { + "x-litellm-api-key": "test-key-123", + "Authorization": "Bearer auth-key-456", + } + result = get_litellm_virtual_key(mock_request) + assert result == "Bearer test-key-123" + + def test_encode_bedrock_runtime_modelid_arn(): # Test application-inference-profile ARN endpoint = "model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse" diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 5baddde6b39..fd02c3bbc65 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -25,7 +25,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( team_membership_reservation_cache_key, ) from litellm.proxy.spend_tracking.budget_reservation import ( - DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, _get_team_member_budget_counter, estimate_request_input_cost, estimate_request_max_cost, @@ -197,9 +196,11 @@ GEMINI_SYSTEM_PARTS: Final = ("Answer tersely", "Use British spelling") ( ({"generationConfig": {"maxOutputTokens": 5}}, {"max_tokens": 5}), ({"generationConfig": {"maxOutputTokens": 5, "candidateCount": 3}}, {"max_tokens": 5, "n": 3}), - ({"generation_config": {"max_output_tokens": 5, "candidate_count": 3}}, {"n": 3}), - ({"generationConfig": {"candidate_count": 3}}, {"n": 3}), - ({}, {}), + ({"generationConfig": {"maxOutputTokens": 5, "candidate_count": 3}}, {"max_tokens": 5, "n": 3}), + ( + {"generationConfig": {"maxOutputTokens": 5}, "generation_config": {"candidate_count": 3}}, + {"max_tokens": 5, "n": 3}, + ), ), ) @pytest.mark.parametrize("system_texts", (GEMINI_SYSTEM_PARTS[:1], GEMINI_SYSTEM_PARTS)) @@ -239,6 +240,7 @@ def test_gemini_generate_content_body_reserves_the_prompt_not_the_context_window GEMINI_ROUTE: Final = "/v1beta/models/gemini-3.8-flash:generateContent" GEMINI_TEXT_CONTENT: Final = {"role": "user", "parts": [{"text": GEMINI_PROMPT}]} GEMINI_AUDIO_PART: Final = {"inlineData": {"mimeType": "audio/wav", "data": "UklGRg" * 40_000}} +GEMINI_OUTPUT_CAP: Final = {"generationConfig": {"maxOutputTokens": 5}} def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> float | None: @@ -277,7 +279,9 @@ def _gemini_hold(body: dict[str, object], input_tokens: int | None = None) -> fl def test_gemini_body_the_prompt_count_cannot_see_whole_keeps_the_context_window_hold(body: dict[str, object]) -> None: context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"] assert context_window is not None - assert _gemini_hold(body) == _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window) + assert _gemini_hold({**body, **GEMINI_OUTPUT_CAP}) == _gemini_hold( + {"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}, input_tokens=context_window + ) def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None: @@ -290,10 +294,12 @@ def test_gemini_tool_turns_and_thought_signatures_count_as_prompt() -> None: {"role": "model", "parts": [{"functionCall": {"name": "lookup", "args": {}}, "thoughtSignature": "c2ln"}]}, {"role": "user", "parts": [{"functionResponse": {"name": "lookup", "response": {"answer": "hi"}}}]}, ], + **GEMINI_OUTPUT_CAP, } - text_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}) + text_body: Final = {"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP} + text_hold: Final = _gemini_hold(text_body) tool_turns_hold: Final = _gemini_hold(tool_turns) - context_window_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}, input_tokens=context_window) + context_window_hold: Final = _gemini_hold(text_body, input_tokens=context_window) assert text_hold is not None and tool_turns_hold is not None and context_window_hold is not None assert text_hold < tool_turns_hold < context_window_hold @@ -328,22 +334,34 @@ def test_gemini_output_fields_on_a_chat_body_leave_its_hold_alone(gemini_output_ assert _chat_hold({**GEMINI_CHAT_BODY, **gemini_output_fields}) == plain_hold -@pytest.mark.parametrize("cap", (0, -1000)) -def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int) -> None: - uncapped_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}) - assert uncapped_hold is not None +@pytest.mark.parametrize( + "generation_config", + ({}, {"candidateCount": 4}, {"maxOutputTokens": 0}, {"maxOutputTokens": -1000}, {"maxOutputTokens": "many"}), +) +def test_a_gemini_call_without_an_output_cap_holds_the_whole_context_window( + generation_config: dict[str, object], +) -> None: + context_window: Final = litellm.get_model_info("gemini-3.8-flash")["max_input_tokens"] + assert context_window is not None + window_filling_chat_hold: Final = estimate_request_max_cost( + request_body=GEMINI_CHAT_BODY, + route="/chat/completions", + llm_router=None, + input_token_counts={"gemini-3.8-flash": context_window}, + ) + assert window_filling_chat_hold is not None assert ( - _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": cap}}) - == uncapped_hold + _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": generation_config}) + == window_filling_chat_hold ) @pytest.mark.parametrize( ("output_fields", "same_hold_fields"), ( - ({"generationConfig": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}), - ({"generation_config": {"maxOutputTokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}), - ({"generation_config": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}), + ({"generationConfig": {"max_output_tokens": 30000}}, {}), + ({"generation_config": {"maxOutputTokens": 30000}}, {}), + ({"generation_config": {"max_output_tokens": 30000}}, {}), ( {"generationConfig": {"maxOutputTokens": 8000}, "generation_config": {"max_output_tokens": 30000}}, {"generationConfig": {"maxOutputTokens": 30000}}, @@ -355,7 +373,7 @@ def test_a_gemini_output_cap_of_zero_or_below_holds_the_default_output(cap: int) ), ), ) -def test_an_output_cap_spelling_v1beta_drops_can_only_raise_the_hold( +def test_an_output_cap_spelled_other_than_max_output_tokens_can_only_raise_the_hold( output_fields: dict[str, object], same_hold_fields: dict[str, object] ) -> None: expected_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **same_hold_fields}) @@ -368,10 +386,12 @@ UNPRICEABLE_COUNTS: Final = (10**400, "9" * 400, 1e308, math.inf) @pytest.mark.parametrize("count", UNPRICEABLE_COUNTS) def test_a_candidate_count_too_large_to_price_holds_one_candidate(count: object) -> None: - one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT]}) + one_candidate_hold: Final = _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], **GEMINI_OUTPUT_CAP}) assert one_candidate_hold is not None assert ( - _gemini_hold({"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"candidateCount": count}}) + _gemini_hold( + {"contents": [GEMINI_TEXT_CONTENT], "generationConfig": {"maxOutputTokens": 5, "candidateCount": count}} + ) == one_candidate_hold ) @@ -412,8 +432,8 @@ def _priced_gemini_hold( ( (WIDE_WINDOW, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}), (WIDE_WINDOW, {"maxOutputTokens": 20000, "candidate_count": 2}, WIDE_WINDOW, {"maxOutputTokens": 40000}), - (WIDE_WINDOW, {"candidateCount": 2}, WIDE_WINDOW, {"maxOutputTokens": 2 * DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK}), - (LOW_OUTPUT_CEILING, {"candidateCount": 4}, {**WIDE_WINDOW, "max_output_tokens": 4000}, {}), + (LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000, "candidateCount": 4}, WIDE_WINDOW, {"maxOutputTokens": 32000}), + (LOW_OUTPUT_CEILING, {"maxOutputTokens": 8000}, WIDE_WINDOW, {"maxOutputTokens": 8000}), ), ) def test_candidates_hold_what_one_candidate_writing_all_their_output_holds( @@ -433,9 +453,10 @@ def test_candidates_hold_what_one_candidate_writing_all_their_output_holds( (WIDE_WINDOW, {"candidateCount": 1e300}), (WIDE_WINDOW, {"maxOutputTokens": 100_000, "candidateCount": 8}), (LOW_OUTPUT_CEILING, {"candidateCount": 1e300}), + (LOW_OUTPUT_CEILING, {"maxOutputTokens": 100_000, "candidateCount": 8}), ), ) -def test_candidates_never_hold_more_than_the_whole_context_window_hold( +def test_a_gemini_hold_never_exceeds_the_whole_context_window_hold( model_info: dict[str, object], generation_config: dict[str, object] ) -> None: context_window_hold: Final = _priced_gemini_hold(model_info, {}, input_tokens=1_000_000) @@ -443,31 +464,14 @@ def test_candidates_never_hold_more_than_the_whole_context_window_hold( assert _priced_gemini_hold(model_info, generation_config) == context_window_hold -def test_candidates_never_hold_less_than_one_candidate() -> None: - narrow_window: Final = {"max_input_tokens": 10, "max_output_tokens": 100_000} - one_candidate_hold: Final = _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000}) - assert one_candidate_hold is not None - assert _priced_gemini_hold(narrow_window, {"maxOutputTokens": 50_000, "candidateCount": 2}) == one_candidate_hold - - NO_WINDOW_MODEL: Final = "vertex_ai/gemini-without-cost-map-entry" @pytest.mark.parametrize( - ("generation_config", "same_hold_config"), - ( - ({"maxOutputTokens": 4000, "candidateCount": 2}, {"maxOutputTokens": 8000}), - ({"maxOutputTokens": 8000, "candidateCount": 4}, {}), - ({"candidateCount": 1_000_000}, {}), - ), + "generation_config", ({}, {"maxOutputTokens": 4000}, {"maxOutputTokens": 8000, "candidateCount": 4}) ) -def test_candidates_of_a_model_with_no_context_window_hold_up_to_one_uncapped_candidate( - generation_config: dict[str, object], same_hold_config: dict[str, object] -) -> None: - no_window: Final = {"max_output_tokens": 100_000} - expected_hold: Final = _priced_gemini_hold(no_window, same_hold_config, model=NO_WINDOW_MODEL) - assert expected_hold is not None - assert _priced_gemini_hold(no_window, generation_config, model=NO_WINDOW_MODEL) == expected_hold +def test_a_gemini_model_with_no_context_window_is_not_held(generation_config: dict[str, object]) -> None: + assert _priced_gemini_hold({"max_output_tokens": 100_000}, generation_config, model=NO_WINDOW_MODEL) is None @pytest.mark.parametrize("count", UNPRICEABLE_COUNTS) @@ -487,18 +491,17 @@ def _group_generate_content_hold(router: Router, generation_config: dict[str, ob @pytest.mark.parametrize( - ("backend_models", "candidates_are_generated"), + "backend_models", ( - (("vertex_ai/claude-haiku-5-5",), False), - (("anthropic/claude-haiku-5-5",), False), - (("gemini/gemini-3.8-flash",), True), - (("vertex_ai/gemini-3.8-flash",), True), - (("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"), True), + ("vertex_ai/claude-haiku-5-5",), + ("anthropic/claude-haiku-5-5",), + ("openai/gpt-5.4-mini",), + ("gemini/gemini-3.8-flash",), + ("vertex_ai/gemini-3.8-flash",), + ("vertex_ai/claude-haiku-5-5", "gemini/gemini-3.8-flash"), ), ) -def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidates( - backend_models: tuple[str, ...], candidates_are_generated: bool -) -> None: +def test_a_candidate_count_is_held_whatever_deployment_serves_the_name(backend_models: tuple[str, ...]) -> None: router: Final = Router( model_list=[ {"model_name": "assistant", "litellm_params": {"model": model, "api_key": "sk-test"}} @@ -510,8 +513,7 @@ def test_a_candidate_count_is_held_only_when_a_deployment_generates_the_candidat assert one_candidate_hold is not None assert three_candidate_hold is not None - assert (three_candidate_hold > one_candidate_hold) is candidates_are_generated - assert three_candidate_hold >= one_candidate_hold + assert three_candidate_hold > one_candidate_hold def test_a_candidate_count_for_a_gemini_name_no_deployment_serves_is_held() -> None: