From af2cf4d70606e09f36074182f0532d2419a14912 Mon Sep 17 00:00:00 2001 From: Kentaro Takiguchi Date: Wed, 4 Nov 2026 05:23:59 +0000 Subject: [PATCH] fix(proxy): strip duplicate and mixed-case proxy auth credentials in clean_headers Google GenAI SDK clients attach x-goog-api-key alongside Authorization or x-litellm-api-key, so stripping only the winning proxy auth header name still leaked the proxy virtual key upstream when forward_llm_provider_auth_headers and forward_client_headers_to_llm_api were both enabled. Empty higher-precedence headers and mixed-case Mapping keys also bypassed litellm_auth_header_name or overwrote upstream credentials with empty values Normalize non-empty headers case-insensitively in litellm_auth_header_name and unify clean_headers provider-auth filtering so empty provider auth headers and any header carrying the bearer-stripped proxy credential are stripped while separately supplied caller provider keys stay forwarded --- litellm/proxy/litellm_pre_call_utils.py | 78 +++++----- .../google_genai/test_google_genai_main.py | 41 +++++- .../unit/proxy/test_litellm_pre_call_utils.py | 138 ++++++++++++++++-- 3 files changed, 205 insertions(+), 52 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 19c41f2cfc1..cc0686a0636 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1511,17 +1511,16 @@ _PROXY_AUTH_HEADER_PRECEDENCE: Final = tuple( ) -def litellm_auth_header_name(headers: Headers, litellm_key_header_name: str | None = None) -> str: - if litellm_key_header_name is not None and litellm_key_header_name in headers: +def _strip_bearer(value: str) -> str: + stripped: Final = value.strip() + return stripped[7:].strip() if stripped[:7].lower() == "bearer " else stripped + + +def litellm_auth_header_name(headers: Mapping[str, str], litellm_key_header_name: str | None = None) -> str: + present: Final = {name.lower() for name, value in headers.items() if value} + if litellm_key_header_name is not None and litellm_key_header_name.lower() in present: return litellm_key_header_name.lower() - return next( - ( - name - for name in _PROXY_AUTH_HEADER_PRECEDENCE - if name in headers and (name != "authorization" or headers[name]) - ), - "x-api-key", - ) + return next((name for name in _PROXY_AUTH_HEADER_PRECEDENCE if name in present), "x-api-key") def forwarded_anthropic_oauth_token(headers: Headers) -> str | None: @@ -1539,11 +1538,11 @@ def forwarded_anthropic_oauth_token(headers: Headers) -> str | None: def clean_headers( - headers: Headers, + headers: Mapping[str, str], litellm_key_header_name: str | None = None, forward_llm_provider_auth_headers: bool = False, authenticated_with_header: str | None = None, -) -> dict[str, str]: +) -> dict[str, str]: # mutable-ok: downstream callers mutate returned headers dict """ Removes litellm api key from headers @@ -1560,38 +1559,35 @@ def clean_headers( from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key from litellm.llms.chatgpt.common_utils import is_chatgpt_oauth_key - clean_headers: Final = {} litellm_key_lower: Final = litellm_key_header_name.lower() if litellm_key_header_name is not None else None - for header, value in headers.items(): - header_lower = header.lower() + auth_header_lower: Final = authenticated_with_header.lower() if authenticated_with_header is not None else None + custom_excluded: Final = frozenset({litellm_key_lower, auth_header_lower}) + provider_excluded: Final = frozenset({"authorization", "x-litellm-api-key", *custom_excluded}) + proxy_auth_credential: Final = ( + next( + ( + credential + for name, raw in headers.items() + if name.lower() == auth_header_lower and (credential := _strip_bearer(raw)) + ), + None, + ) + if auth_header_lower is not None + else None + ) + def _keep(header: str, value: str) -> bool: + header_lower: Final = header.lower() if header_lower == "authorization" and (is_anthropic_oauth_key(value) or is_chatgpt_oauth_key(value)): - if authenticated_with_header is None or authenticated_with_header.lower() != "authorization": - clean_headers[header] = value - continue - # Special handling for x-api-key: forward it based on authenticated_with_header - elif header_lower == "x-api-key": - if forward_llm_provider_auth_headers and ( - authenticated_with_header is None or authenticated_with_header.lower() != "x-api-key" - ): - clean_headers[header] = value - elif forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE: - if litellm_key_lower and header_lower == litellm_key_lower: - continue - if authenticated_with_header is not None and header_lower == authenticated_with_header.lower(): - continue - if header_lower == "authorization": - continue - # Never forward x-litellm-api-key (it's for proxy auth only) - if header_lower == "x-litellm-api-key": - continue - clean_headers[header] = value - # Check if header should be excluded: either in special headers cache or matches custom litellm key - elif header_lower not in _SPECIAL_HEADERS_CACHE and ( - litellm_key_lower is None or header_lower != litellm_key_lower - ): - clean_headers[header] = value - return clean_headers + return auth_header_lower != "authorization" and _strip_bearer(value) != proxy_auth_credential + if forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE: + if header_lower in provider_excluded: + return False + credential: Final = _strip_bearer(value) + return bool(credential) and credential != proxy_auth_credential + return header_lower not in _SPECIAL_HEADERS_CACHE and header_lower not in custom_excluded + + return {header: value for header, value in headers.items() if _keep(header, value)} def _request_fields(value: object) -> Mapping[str, object]: diff --git a/tests/unit/google_genai/test_google_genai_main.py b/tests/unit/google_genai/test_google_genai_main.py index 0f59daee1b6..a80e8b21515 100644 --- a/tests/unit/google_genai/test_google_genai_main.py +++ b/tests/unit/google_genai/test_google_genai_main.py @@ -734,6 +734,30 @@ async def test_proxy_forwarded_headers_reach_generate_content_without_leaking_pr ("incoming", "expected_upstream_key"), [ ({"x-goog-api-key": "sk-litellm-virtual-key", "x-cost-center": "core-platform"}, _UPSTREAM_KEY), + ( + { + "authorization": "Bearer sk-litellm-virtual-key", + "x-goog-api-key": "sk-litellm-virtual-key", + "x-cost-center": "core-platform", + }, + _UPSTREAM_KEY, + ), + ( + { + "x-api-key": "", + "x-goog-api-key": "sk-litellm-virtual-key", + "x-cost-center": "core-platform", + }, + _UPSTREAM_KEY, + ), + ( + { + "authorization": "Bearer sk-litellm-virtual-key", + "x-goog-api-key": "", + "x-cost-center": "core-platform", + }, + _UPSTREAM_KEY, + ), ( { "x-litellm-api-key": "sk-litellm-virtual-key", @@ -742,8 +766,23 @@ async def test_proxy_forwarded_headers_reach_generate_content_without_leaking_pr }, "AIza-caller-key", ), + ( + { + "authorization": "Bearer sk-litellm-virtual-key", + "x-goog-api-key": "AIza-caller-key", + "x-cost-center": "core-platform", + }, + "AIza-caller-key", + ), + ], + ids=[ + "proxy-key-in-goog-header", + "sdk-dual-bearer-and-goog-proxy-key", + "empty-higher-precedence-header", + "empty-goog-header-beside-bearer-key", + "caller-key-beside-litellm-key", + "caller-key-beside-bearer-key", ], - ids=["proxy-key-in-goog-header", "caller-key-beside-proxy-key"], ) async def test_provider_auth_forwarding_never_sends_proxy_key_to_generate_content( header_capture_wire: _WireCapture, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 074aa0c34e8..14e92cc343e 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -6453,25 +6453,33 @@ def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth ("headers", "litellm_key_header_name", "expected"), [ ({"x-litellm-api-key": "sk-proxy", "authorization": "Bearer sk-other"}, None, "x-litellm-api-key"), + ({"x-litellm-api-key": "", "x-goog-api-key": "sk-proxy"}, None, "x-goog-api-key"), ({"authorization": "Bearer sk-proxy", "x-goog-api-key": "AIza-caller"}, None, "authorization"), ({"authorization": "", "x-goog-api-key": "sk-proxy"}, None, "x-goog-api-key"), ({"api-key": "sk-proxy", "x-api-key": "sk-ant-caller"}, None, "api-key"), + ({"x-api-key": "", "x-goog-api-key": "sk-proxy"}, None, "x-goog-api-key"), ({"x-api-key": "sk-proxy", "x-goog-api-key": "AIza-caller"}, None, "x-api-key"), ({"x-goog-api-key": "sk-proxy"}, None, "x-goog-api-key"), + ({"X-Goog-Api-Key": "sk-proxy"}, None, "x-goog-api-key"), ({"ocp-apim-subscription-key": "sk-proxy"}, None, "ocp-apim-subscription-key"), - ({"x-custom-key": "sk-proxy", "authorization": "Bearer sk-other"}, "X-Custom-Key", "x-custom-key"), + ({"X-Custom-Key": "sk-proxy", "authorization": "Bearer sk-other"}, "x-custom-key", "x-custom-key"), + ({"x-custom-key": "", "x-goog-api-key": "sk-proxy"}, "x-custom-key", "x-goog-api-key"), ({"authorization": "Bearer sk-proxy"}, "x-custom-key", "authorization"), ({"content-type": "application/json"}, None, "x-api-key"), ], ids=[ "litellm-key-beats-authorization", + "empty-litellm-key-skipped", "authorization-beats-goog", "empty-authorization-skipped", "azure-beats-anthropic", + "empty-anthropic-skipped", "anthropic-beats-goog", "goog", + "goog-mixed-case-dict", "azure-apim", - "custom-header-wins-when-present", + "custom-header-mixed-case-dict", + "empty-custom-header-skipped", "custom-header-absent", "no-auth-header", ], @@ -6479,6 +6487,7 @@ def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth def test_litellm_auth_header_name_follows_user_api_key_auth_precedence( headers: Mapping[str, str], litellm_key_header_name: str | None, expected: str ) -> None: + assert litellm_auth_header_name(headers, litellm_key_header_name) == expected assert litellm_auth_header_name(Headers(headers), litellm_key_header_name) == expected @@ -6489,29 +6498,88 @@ def test_litellm_auth_header_name_follows_user_api_key_auth_precedence( {"x-goog-api-key": "sk-proxy-virtual-key", "x-cost-center": "core-platform"}, {"x-cost-center": "core-platform"}, ), + ( + {"X-Goog-Api-Key": "sk-proxy-virtual-key", "x-cost-center": "core-platform"}, + {"x-cost-center": "core-platform"}, + ), + ( + { + "authorization": "Bearer sk-proxy-virtual-key", + "x-goog-api-key": "sk-proxy-virtual-key", + "x-cost-center": "core-platform", + }, + {"x-cost-center": "core-platform"}, + ), + ( + { + "x-litellm-api-key": "sk-proxy-virtual-key", + "x-goog-api-key": "sk-proxy-virtual-key", + "x-cost-center": "core-platform", + }, + {"x-cost-center": "core-platform"}, + ), + ( + { + "x-api-key": "", + "x-goog-api-key": "sk-proxy-virtual-key", + "x-cost-center": "core-platform", + }, + {"x-cost-center": "core-platform"}, + ), ( {"x-litellm-api-key": "sk-proxy-virtual-key", "x-goog-api-key": "AIza-caller-key"}, {"x-goog-api-key": "AIza-caller-key"}, ), + ( + {"authorization": "Bearer sk-proxy-virtual-key", "x-goog-api-key": "AIza-caller-key"}, + {"x-goog-api-key": "AIza-caller-key"}, + ), ( {"api-key": "sk-proxy-virtual-key", "x-goog-api-key": "AIza-caller-key"}, {"x-goog-api-key": "AIza-caller-key"}, ), ], - ids=["goog-was-proxy-auth", "goog-beside-litellm-key", "goog-beside-azure-proxy-key"], + ids=[ + "goog-was-proxy-auth", + "goog-mixed-case-was-proxy-auth", + "duplicate-proxy-key-in-bearer-and-goog", + "duplicate-proxy-key-in-litellm-and-goog", + "empty-provider-header-and-goog-proxy-auth", + "goog-beside-litellm-key", + "goog-beside-bearer-proxy-key", + "goog-beside-azure-proxy-key", + ], ) def test_clean_headers_byok_strips_only_the_header_used_for_proxy_auth( headers: Mapping[str, str], expected: Mapping[str, str] ) -> None: + assert ( + clean_headers( + headers, + forward_llm_provider_auth_headers=True, + authenticated_with_header=litellm_auth_header_name(headers), + ) + == expected + ) incoming: Final = Headers(headers) - - result: Final = clean_headers( - incoming, - forward_llm_provider_auth_headers=True, - authenticated_with_header=litellm_auth_header_name(incoming), + assert ( + clean_headers( + incoming, + forward_llm_provider_auth_headers=True, + authenticated_with_header=litellm_auth_header_name(incoming), + ) + == expected ) - assert result == expected + +def test_clean_headers_strips_authenticated_custom_header_without_litellm_key_name() -> None: + cleaned: Final = clean_headers( + {"X-Custom-Proxy-Key": "sk-proxy-virtual-key", "x-cost-center": "core-platform"}, + forward_llm_provider_auth_headers=True, + authenticated_with_header="x-custom-proxy-key", + ) + + assert cleaned == {"x-cost-center": "core-platform"} def test_clean_headers_byok_strips_custom_litellm_key_header_named_x_api_key() -> None: @@ -8054,6 +8122,38 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_ FORWARD_PROVIDER_AUTH_HEADERS, {}, ), + ( + {"X-Goog-Api-Key": "sk-virtual-key", "x-cost-center": "core-platform"}, + FORWARD_PROVIDER_AUTH_HEADERS, + {}, + ), + ( + { + "authorization": "Bearer sk-virtual-key", + "x-goog-api-key": "sk-virtual-key", + "x-cost-center": "core-platform", + }, + FORWARD_PROVIDER_AUTH_HEADERS, + {}, + ), + ( + { + "Authorization": "Bearer sk-virtual-key", + "X-Goog-Api-Key": "sk-virtual-key", + "x-cost-center": "core-platform", + }, + FORWARD_PROVIDER_AUTH_HEADERS, + {}, + ), + ( + { + "x-api-key": "", + "x-goog-api-key": "sk-virtual-key", + "x-cost-center": "core-platform", + }, + FORWARD_PROVIDER_AUTH_HEADERS, + {}, + ), ( { "x-litellm-api-key": "Bearer sk-virtual-key", @@ -8068,8 +8168,26 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_ {**FORWARD_PROVIDER_AUTH_HEADERS, "litellm_key_header_name": "x-api-key"}, {}, ), + ( + { + "x-custom-proxy-key": "sk-virtual-key", + "x-goog-api-key": "AIza-caller-key", + "x-cost-center": "core-platform", + }, + {**FORWARD_PROVIDER_AUTH_HEADERS, "litellm_key_header_name": "x-custom-proxy-key"}, + {"x-goog-api-key": "AIza-caller-key"}, + ), + ], + ids=[ + "goog-was-proxy-auth", + "goog-mixed-case-was-proxy-auth", + "duplicate-proxy-key-in-bearer-and-goog", + "duplicate-proxy-key-mixed-case", + "empty-higher-precedence-header", + "goog-beside-litellm-key", + "custom-litellm-key-header", + "custom-litellm-key-header-preserves-goog-byok", ], - ids=["goog-was-proxy-auth", "goog-beside-litellm-key", "custom-litellm-key-header"], ) async def test_add_litellm_data_to_request_never_forwards_the_header_used_for_proxy_auth( request_headers: Mapping[str, str],