From 728b73d1b37efe118751752bfeedcf8d6e21faf4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:04:27 -0700 Subject: [PATCH] fix(proxy): strip only the authenticating secret on credential-less Vertex passthrough PR #38114 dropped whichever header user_api_key_auth would read the caller's key from, by precedence. Under custom_auth, JWT auth, or no master key that header is the caller's own Google token, so the bring-your-own-credentials Vertex branch answered 401 to every valid request. A header value is now dropped only when it is the master key or when its hash is the api_key that authenticated the request, so a Google token that auth never consumed keeps flowing while a LiteLLM key still never reaches Google. test_passthrough_post_call_guardrails.py no longer plants a MagicMock proxy_server module in sys.modules at import, which poisoned sibling tests that read module globals at call time. --- bootstrap.log | 271 ++++++++++++++++++ .../llm_passthrough_endpoints.py | 116 ++++---- .../test_llm_pass_through_endpoints.py | 83 +++++- .../test_passthrough_post_call_guardrails.py | 16 -- .../test_vertex_passthrough_load_balancing.py | 3 + 5 files changed, 398 insertions(+), 91 deletions(-) create mode 100644 bootstrap.log diff --git a/bootstrap.log b/bootstrap.log new file mode 100644 index 00000000000..864cbf2d57e --- /dev/null +++ b/bootstrap.log @@ -0,0 +1,271 @@ +uv sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev +Using CPython 3.12.13 +Creating virtual environment at: .venv + Building litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix + Building litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras + Building litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise + Built litellm-enterprise @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise + Built litellm-proxy-extras @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras + Built litellm @ file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix +Prepared 3 packages in 1m 00s +Installed 229 packages in 637ms + + a2a-sdk==1.1.0 + + aiohappyeyeballs==2.6.1 + + aiohttp==3.14.3 + + aiologic==0.17.0 + + aiosignal==1.4.0 + + annotated-doc==0.0.4 + + annotated-types==0.7.0 + + anyio==4.13.0 + + apscheduler==3.11.2 + + asgiref==3.11.1 + + attrs==26.1.0 + + azure-core==1.41.0 + + azure-identity==1.25.2 + + azure-storage-blob==12.28.0 + + backoff==2.2.1 + + basedpyright==1.39.7 + + bidict==0.23.1 + + blinker==1.9.0 + + boto3==1.43.1 + + botocore==1.43.6 + + botocore-stubs==1.43.14 + + brotli==1.2.0 + + certifi==2026.4.22 + + cffi==2.0.0 + + chardet==7.4.3 + + charset-normalizer==3.4.7 + + click==8.3.3 + + configargparse==1.7.5 + + coverage==7.14.0 + + croniter==6.2.2 + + cryptography==50.0.0 + + culsans==0.11.0 + + deprecated==1.3.1 + + diff-cover==9.7.2 + + distro==1.9.0 + + dnspython==2.8.0 + + email-validator==2.3.0 + + execnet==2.1.2 + + expression==5.6.0 + + fakeredis==2.34.1 + + fastapi==0.136.3 + + fastapi-offline==1.7.6 + + fastapi-sso==0.19.0 + + fastuuid==0.14.0 + + filelock==3.29.0 + + flask==3.1.3 + + flask-cors==6.0.2 + + flask-login==0.6.3 + + frozenlist==1.8.0 + + fsspec==2026.4.0 + + gevent==25.9.1 + + geventhttpclient==2.3.9 + + google-api-core==2.30.3 + + google-auth==2.52.0 + + googleapis-common-protos==1.75.0 + + granian==2.7.4 + + greenlet==3.5.0 + + grpcio==1.78.0 + + gunicorn==23.0.0 + + h11==0.16.0 + + h2==4.4.1 + + hf-xet==1.5.0 + + hiredis==3.4.0 + + hpack==4.2.0 + + httpcore==1.0.9 + + httpx==0.28.1 + + httpx-sse==0.4.3 + + huggingface-hub==1.14.0 + + hypercorn==0.17.3 + + hyperframe==6.1.0 + + idna==3.15 + + importlib-metadata==8.5.0 + + iniconfig==2.3.0 + + inquirerpy==0.3.4 + + isodate==0.7.2 + + itsdangerous==2.2.0 + + jaraco-classes==3.4.0 + + jaraco-context==6.1.2 + + jaraco-functools==4.6.0 + + jinja2==3.1.6 + + jiter==0.14.0 + + jmespath==1.1.0 + + json-rpc==1.15.0 + + jsonschema==4.26.0 + + jsonschema-path==0.3.4 + + jsonschema-specifications==2025.9.1 + + keyring==25.7.0 + + langfuse==2.59.7 + + lazy-object-proxy==1.12.0 + + litellm==1.100.0 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix) + + litellm-enterprise==0.1.60 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/enterprise) + + litellm-proxy-extras==0.4.89 (from file:///private/tmp/claude-501/-Users-mateo-Development-litellm/5f605a1a-6687-495a-8f8f-e1f96a73b6da/scratchpad/vertexpt-fix/litellm-proxy-extras) + + locust==2.45.0 + + markdown-it-py==4.2.0 + + markupsafe==3.0.3 + + mcp==1.28.1 + + mdurl==0.1.2 + + mirakuru==3.0.2 + + more-itertools==11.0.2 + + msal==1.37.0 + + msal-extensions==1.3.1 + + msgpack==1.2.1 + + multidict==6.7.1 + + nodeenv==1.10.0 + + nodejs-wheel-binaries==24.16.0 + + oauthlib==3.3.1 + + openai==2.33.0 + + openapi-core==0.22.0 + + openapi-schema-validator==0.6.3 + + openapi-spec-validator==0.7.2 + + opentelemetry-api==1.28.0 + + opentelemetry-exporter-otlp==1.28.0 + + opentelemetry-exporter-otlp-proto-common==1.28.0 + + opentelemetry-exporter-otlp-proto-grpc==1.28.0 + + opentelemetry-exporter-otlp-proto-http==1.28.0 + + opentelemetry-instrumentation==0.49b0 + + opentelemetry-instrumentation-asgi==0.49b0 + + opentelemetry-instrumentation-fastapi==0.49b0 + + opentelemetry-proto==1.28.0 + + opentelemetry-sdk==1.28.0 + + opentelemetry-semantic-conventions==0.49b0 + + opentelemetry-util-http==0.49b0 + + orjson==3.11.6 + + packaging==26.2 + + parameterized==0.9.0 + + pathable==0.4.4 + + pfzy==0.3.4 + + playwright==1.61.0 + + pluggy==1.6.0 + + polars==1.38.1 + + polars-runtime-32==1.38.1 + + port-for==1.0.0 + + priority==2.0.0 + + prisma==0.11.0 + + prometheus-client==0.20.0 + + prompt-toolkit==3.0.52 + + propcache==0.5.2 + + proto-plus==1.28.0 + + protobuf==5.29.6 + + psutil==7.2.2 + + psycopg==3.3.3 + + psycopg-binary==3.3.3 + + pyasn1==0.6.4 + + pyasn1-modules==0.4.2 + + pycparser==3.0 + + pydantic==2.13.4 + + pydantic-core==2.46.4 + + pydantic-settings==2.14.2 + + pyee==13.0.1 + + pygments==2.20.0 + + pyjwt==2.13.0 + + pynacl==1.6.2 + + pyroscope-io==0.8.16 + + pytest==9.0.3 + + pytest-asyncio==1.3.0 + + pytest-cov==5.0.0 + + pytest-mock==3.15.1 + + pytest-postgresql==7.0.2 + + pytest-recording==0.13.4 + + pytest-rerunfailures==15.1 + + pytest-timeout==2.4.0 + + pytest-xdist==3.8.0 + + python-dateutil==2.9.0.post0 + + python-dotenv==1.2.2 + + python-engineio==4.13.3 + + python-multipart==0.0.32 + + python-socketio==5.16.3 + + pyyaml==6.0.3 + + pyzmq==27.1.0 + + redis==5.3.1 + + referencing==0.36.2 + + regex==2026.5.9 + + requests==2.34.0 + + requests-mock==1.12.1 + + responses==0.26.0 + + respx==0.22.0 + + restrictedpython==8.1 + + rfc3339-validator==0.1.4 + + rich==13.9.4 + + rpds-py==0.30.0 + + rq==2.7.0 + + ruff==0.15.3 + + s3transfer==0.17.0 + + shellingham==1.5.4 + + simple-websocket==1.1.0 + + six==1.17.0 + + sniffio==1.3.1 + + sortedcontainers==2.4.0 + + soundfile==0.12.1 + + sse-starlette==3.4.4 + + starlette==1.3.1 + + tiktoken==0.12.0 + + tokenizers==0.23.1 + + tomlkit==0.15.0 + + tqdm==4.67.3 + + typer==0.25.1 + + types-awscrt==0.34.1 + + types-boto3==1.43.30 + + types-boto3-bedrock==1.43.26 + + types-boto3-bedrock-agent==1.43.0 + + types-boto3-bedrock-runtime==1.43.30 + + types-boto3-kms==1.43.12 + + types-boto3-s3==1.43.14 + + types-boto3-sagemaker-runtime==1.43.29 + + types-boto3-sts==1.43.0 + + types-cffi==2.0.0.20260508 + + types-pyopenssl==24.1.0.20240722 + + types-pyyaml==6.0.12.20250915 + + types-redis==4.6.0.20241004 + + types-requests==2.32.4.20260107 + + types-s3transfer==0.16.0 + + types-setuptools==75.8.0.20250225 + + typing-extensions==4.15.0 + + typing-inspection==0.4.2 + + tzlocal==5.3.1 + + urllib3==2.7.0 + + uvicorn==0.51.0 + + uvloop==0.21.0 + + vcrpy==8.2.1 + + wcwidth==0.7.0 + + websocket-client==1.9.0 + + websockets==15.0.1 + + werkzeug==3.1.8 + + wrapt==1.17.3 + + wsproto==1.3.2 + + yarl==1.23.0 + + zipp==3.23.1 + + zope-event==6.2 + + zope-interface==8.5 +uv run --no-sync python scripts/prisma_generate_if_needed.py +Prisma schema loaded from litellm/proxy/schema.prisma + +Some types are disabled by default due to being incompatible with Mypy, it is highly recommended +to use Pyright instead and configure Prisma Python to use recursive types. To re-enable certain types: + +generator client { + provider = "prisma-client-py" + recursive_type_depth = -1 +} + +If you need to use Mypy, you can also disable this message by explicitly setting the default value: + +generator client { + provider = "prisma-client-py" + recursive_type_depth = 5 +} + +For more information see: https://prisma-client-py.readthedocs.io/en/stable/reference/limitations/#default-type-limitations + + +Warning: The binaryTargets option is not officially supported by Prisma Client Python. + +✔ Generated Prisma Client Python (v0.11.0) to ./.venv/lib/python3.12/site-packages/prisma in 401ms + +cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund +npm warn deprecated node-domexception@1.0.0: Use your platform's native DOMException instead + +added 738 packages in 9s +bootstrap: copied .env from /Users/mateo/Development/litellm +bootstrap: done diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 850b50227f4..0234e38c921 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -6,6 +6,7 @@ Provider-specific Pass-Through Endpoints Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. """ +import hmac import json import os import re @@ -48,7 +49,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( create_websocket_passthrough_route, websocket_passthrough_request, ) -from litellm.proxy.utils import is_known_model +from litellm.proxy.utils import hash_token, is_known_model from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, assert_user_can_access_vector_store, @@ -1791,28 +1792,17 @@ _HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"} ) -_VERTEX_CALLER_KEY_HEADER_PRECEDENCE: Final = ( - SpecialHeaders.custom_litellm_api_key.value.lower(), - SpecialHeaders.openai_authorization.value.lower(), - SpecialHeaders.azure_authorization.value.lower(), - SpecialHeaders.anthropic_authorization.value.lower(), - SpecialHeaders.google_ai_studio_authorization.value.lower(), - SpecialHeaders.azure_apim_authorization.value.lower(), -) - _MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key" -def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tuple[str, ...]]: - """Operator-configured caller-key header names, as (override, pass_through). +def _operator_configured_caller_key_header_names() -> tuple[str, ...]: + """Operator-configured caller-key header names. - ``user_api_key_auth`` accepts the caller's key from two runtime-configured - header sources beyond the built-in ones, at opposite ends of its precedence. - ``general_settings.litellm_key_header_name`` overrides every built-in source - (it replaces the resolved key after ``get_api_key`` runs), so it is highest - precedence. Each ``general_settings.pass_through_endpoints`` entry's - ``headers.litellm_user_api_key`` is checked last inside ``get_api_key``, so it - is lowest. Google never consumes either, so both are also dropped by name. + ``user_api_key_auth`` also reads the caller's key from + ``general_settings.litellm_key_header_name`` and from each + ``general_settings.pass_through_endpoints`` entry's + ``headers.litellm_user_api_key``. Google never consumes either, so both are + dropped by name. """ from litellm.proxy.proxy_server import general_settings @@ -1829,42 +1819,34 @@ def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tup if isinstance(headers, dict) and isinstance(headers.get("litellm_user_api_key"), str) ) ) - return override, pass_through + return override + pass_through -def _authenticated_caller_key_values(request: Request) -> frozenset[str]: - """The value ``user_api_key_auth`` would accept as this caller's LiteLLM key. +def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool: + """Whether a header value is a LiteLLM secret that authenticated this caller. - The Vertex route authenticates through ``Depends(user_api_key_auth)``, which - resolves the key by precedence, matched here exactly. The ``/vertex_ai`` route - is a mapped pass-through route, so a header literally named - ``litellm_user_api_key`` overrides every other source (``user_api_key_auth`` - applies it last), making it highest precedence. Then an operator - ``litellm_key_header_name``, then the built-in headers in ``get_api_key`` order, - then a ``pass_through_endpoints`` ``litellm_user_api_key`` header which - ``get_api_key`` checks last. Some of those headers (``Authorization``, - ``x-goog-api-key``) are also kept as genuine bring-your-own Google credentials, - so returning only the value that actually authenticated lets the filter strip - that value wherever it appears while leaving a real Google credential in place. - An empty set means no caller key was found, so nothing is value-stripped. + That is the proxy master key, or a LiteLLM-issued key whose ``hash_token`` is + the ``api_key`` ``user_api_key_auth`` resolved (a DB virtual key, or a custom + auth returning the key it validated). A value auth only echoed unhashed (a + custom auth's own identifier, the no-master-key path returning the raw + header) or never consumed (JWT auth resolves ``api_key=None``) is not a + LiteLLM secret, so a bring-your-own Google credential in that position is + still forwarded. """ - incoming: Final = _safe_get_request_headers(request) - override_headers, pass_through_headers = _operator_configured_caller_key_header_names() - ordered_names: Final = ( - (_MAPPED_ROUTE_CALLER_KEY_HEADER,) - + override_headers - + _VERTEX_CALLER_KEY_HEADER_PRECEDENCE - + pass_through_headers - ) - present_values: Final = (incoming[name] for name in ordered_names if incoming.get(name)) - authenticated_key: Final = next( - (stripped for value in present_values if (stripped := _normalize_credential_value(value))), - "", - ) - return frozenset({authenticated_key}) if authenticated_key else frozenset() + from litellm.proxy.proxy_server import master_key + + normalized: Final = _normalize_credential_value(value) + if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()): + return True + authenticated_key: Final = user_api_key_dict.api_key + if authenticated_key is None: + return False + return hmac.compare_digest(hash_token(normalized).encode(), authenticated_key.encode()) -def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) -> Mapping[str, str]: +def _forwarded_headers_for_credentialless_vertex_passthrough( + request: Request, user_api_key_dict: UserAPIKeyAuth +) -> Mapping[str, str]: """ Header set to forward on the bring-your-own-credentials Vertex passthrough branch, used when the proxy has no Vertex credential configured. @@ -1878,31 +1860,25 @@ def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) - ``api-key`` / ``x-api-key`` / ``Ocp-Apim-Subscription-Key``, plus the mapped pass-through ``litellm_user_api_key`` header and any operator-configured ``litellm_key_header_name`` / ``pass_through_endpoints`` key header) are dropped - by name. ``Authorization`` and ``x-goog-api-key`` may - instead carry a genuine bring-your-own Google credential, so they are kept - unless their value is the caller's authenticated LiteLLM key, which is dropped - by value (normalizing any ``Bearer`` / ``Basic`` / ``AWS4`` auth-scheme prefix - the same way authentication does). Because the value that authenticated is - resolved by the same precedence ``user_api_key_auth`` uses, a virtual key sent - only in ``x-goog-api-key`` (or in an operator-configured key header) is dropped - too, while a real Google key in ``x-goog-api-key`` alongside a virtual key in a - higher-precedence header is preserved. When neither a surviving - ``Authorization`` nor ``x-goog-api-key`` remains the request is rejected so the - virtual key cannot leak upstream. + by name. ``Authorization`` and ``x-goog-api-key`` may instead carry a genuine + bring-your-own Google credential, so they are kept unless their value is the + LiteLLM secret that authenticated this caller (``_is_authenticated_caller_secret``), + which is dropped by value wherever it appears. Matching on what actually + authenticated, rather than on header precedence, keeps a caller's own Google + token in ``Authorization`` flowing when a custom auth or JWT authenticated + them without consuming it. When neither a surviving ``Authorization`` nor + ``x-goog-api-key`` remains the request is rejected so the virtual key cannot + leak upstream. """ incoming: Final = _safe_get_request_headers(request) - caller_key_values: Final = _authenticated_caller_key_values(request) - override_headers, pass_through_headers = _operator_configured_caller_key_header_names() - never_forwarded: Final = ( - _HEADERS_NEVER_FORWARDED_TO_VERTEX.union((_MAPPED_ROUTE_CALLER_KEY_HEADER,)) - .union(override_headers) - .union(pass_through_headers) + never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union( + (_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names()) ) forwarded: Final = MappingProxyType( { name: value for name, value in incoming.items() - if name not in never_forwarded and _normalize_credential_value(value) not in caller_key_values + if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict) } ) if "authorization" not in forwarded and "x-goog-api-key" not in forwarded: @@ -1918,6 +1894,7 @@ async def _prepare_vertex_auth_headers( vertex_location: str | None, base_target_url: str | None, get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler, + user_api_key_dict: UserAPIKeyAuth, ) -> tuple[Mapping[str, str], str | None, bool, str | None, str | None]: """ Prepare authentication headers for Vertex AI pass-through requests. @@ -1930,6 +1907,8 @@ async def _prepare_vertex_auth_headers( vertex_location: Vertex location base_target_url: Base URL for the Vertex AI service get_vertex_pass_through_handler: Handler for the specific Vertex AI service + user_api_key_dict: The caller's resolved authentication, so only the secret that + authenticated them is stripped on the credential-less branch Returns: Tuple containing: @@ -1944,7 +1923,7 @@ async def _prepare_vertex_auth_headers( # Use headers from the incoming request if no vertex credentials are found if (vertex_credentials is None or vertex_credentials.vertex_project is None) and router_credentials is None: - headers = _forwarded_headers_for_credentialless_vertex_passthrough(request) + headers = _forwarded_headers_for_credentialless_vertex_passthrough(request, user_api_key_dict) headers_passed_through = True verbose_proxy_logger.debug( "default_vertex_config not set, forwarding caller-provided headers %s", tuple(headers.keys()) @@ -2104,6 +2083,7 @@ async def _base_vertex_proxy_route( vertex_location=vertex_location, base_target_url=base_target_url, get_vertex_pass_through_handler=get_vertex_pass_through_handler, + user_api_key_dict=user_api_key_dict, ) if base_target_url is None: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index a1725005dbf..27f2521702a 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -16,6 +16,7 @@ from starlette.datastructures import FormData import litellm +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, @@ -592,7 +593,7 @@ class TestVertexAIPassThroughHandler: "method": "POST", "path": endpoint, "headers": [ - (b"authorization", b"Bearer test-creds"), + (b"authorization", b"Bearer sk-test-creds"), ], } ) @@ -617,7 +618,7 @@ class TestVertexAIPassThroughHandler: ): mock_ensure_token.return_value = ("test-auth-header", test_project) mock_get_token.return_value = (test_token, "") - mock_auth.return_value = MagicMock() + mock_auth.return_value = UserAPIKeyAuth(api_key="sk-test-creds") with pytest.raises(HTTPException) as exc_info: await vertex_proxy_route( @@ -3342,14 +3343,14 @@ class TestVertexRawPredictStreamingClassification: ), mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), mock.patch(f"{module}.get_litellm_virtual_key", return_value="Bearer test-key"), - mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value={"api_key": "test-key"})), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=UserAPIKeyAuth(api_key="test-key"))), mock.patch(f"{module}.get_vertex_pass_through_handler", return_value=mock_handler), ): await vertex_proxy_route( endpoint=endpoint, request=request, fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(token="test-key"), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), ) assert captured, "create_pass_through_route was never called" @@ -3466,6 +3467,12 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: genuine bring-your-own Google credential that must still pass through. The by-value strip also covers a virtual key sent in the operator-configured ``general_settings.litellm_key_header_name``, whatever that header is named. + + The by-value strip keys off what actually authenticated the caller (the + master key, or the LiteLLM key whose hash ``user_api_key_auth`` resolved as + ``api_key``), never off header precedence: a custom auth or JWT that + authenticated the caller without consuming ``Authorization`` leaves the + caller's own Google token there, and it must keep flowing. """ VKEY = "sk-litellm-victim-key" @@ -3475,8 +3482,12 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: ) async def _run( - self, monkeypatch, headers: list[tuple[bytes, bytes]] + self, + monkeypatch, + headers: list[tuple[bytes, bytes]], + authenticated: UserAPIKeyAuth | None = None, ) -> tuple[HTTPException | None, dict | None]: + caller: Final = authenticated if authenticated is not None else UserAPIKeyAuth(api_key=self.VKEY) from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( PassthroughEndpointRouter, ) @@ -3509,7 +3520,7 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: raised: HTTPException | None = None with ( mock.patch(f"{module}.create_pass_through_route", side_effect=fake_create_pass_through_route), - mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=UserAPIKeyAuth(token="hashed"))), + mock.patch(f"{module}.user_api_key_auth", new=AsyncMock(return_value=caller)), mock.patch(f"{module}.get_vertex_pass_through_handler", return_value=mock_handler), ): try: @@ -3517,7 +3528,7 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: endpoint=self.ENDPOINT, request=request, fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(token="hashed"), + user_api_key_dict=caller, ) except HTTPException as exc: raised = exc @@ -3767,6 +3778,64 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: assert forwarded is None, "a virtual key in the mapped-route litellm_user_api_key header must be dropped, not forwarded" assert raised is not None and raised.status_code == 401 + GOOGLE_OAUTH_TOKEN = "ya29.byo-google-oauth-token" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "authenticated", + [ + pytest.param( + UserAPIKeyAuth(api_key="best-api-key-ever", user_role=LitellmUserRoles.PROXY_ADMIN), + id="custom-auth-returning-its-own-identifier", + ), + pytest.param(UserAPIKeyAuth(api_key=None, user_id="jwt-subject"), id="jwt-auth"), + pytest.param(UserAPIKeyAuth(api_key=GOOGLE_OAUTH_TOKEN), id="no-master-key-echoes-raw-header"), + ], + ) + async def test_google_token_in_authorization_is_forwarded_when_auth_did_not_consume_it( + self, monkeypatch, authenticated: UserAPIKeyAuth + ): + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", f"Bearer {self.GOOGLE_OAUTH_TOKEN}".encode()), + (b"content-type", b"application/json"), + ], + authenticated=authenticated, + ) + assert raised is None, f"the caller's own Google token must not be mistaken for a LiteLLM key: {raised}" + assert forwarded is not None + assert forwarded.get("authorization") == f"Bearer {self.GOOGLE_OAUTH_TOKEN}" + + @pytest.mark.asyncio + async def test_master_key_in_authorization_alone_is_rejected(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") + raised, forwarded = await self._run( + monkeypatch, + [(b"authorization", b"Bearer sk-master-1234"), (b"content-type", b"application/json")], + authenticated=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert forwarded is None, "the master key must never reach the upstream forwarder" + assert raised is not None and raised.status_code == 401 + + @pytest.mark.asyncio + async def test_master_key_is_stripped_and_byo_x_goog_api_key_forwards(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-master-1234") + raised, forwarded = await self._run( + monkeypatch, + [ + (b"authorization", b"Bearer sk-master-1234"), + (b"x-goog-api-key", b"AIza-real-google-api-key"), + (b"content-type", b"application/json"), + ], + authenticated=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert raised is None + assert forwarded is not None + assert forwarded.get("x-goog-api-key") == "AIza-real-google-api-key" + assert "authorization" not in forwarded + assert "sk-master-1234" not in " ".join(f"{name}:{value}" for name, value in forwarded.items()) + class TestGetAzureAISearchIndexFromEndpoint: """The operable index is only the segment right after ``indexes``. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py index a48e9e9e17f..9d1975513a1 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_post_call_guardrails.py @@ -6,7 +6,6 @@ non-streaming pass-through responses. Addresses issue #20270. """ import json -import sys from contextlib import ExitStack from unittest.mock import AsyncMock, MagicMock, patch @@ -66,21 +65,6 @@ def _make_mock_request(): return mock_request -def _ensure_proxy_server_mock(): - """Insert a mock proxy_server module if the real one can't import.""" - key = "litellm.proxy.proxy_server" - if key not in sys.modules: - mock_mod = MagicMock() - mock_mod.proxy_logging_obj = MagicMock() - sys.modules[key] = mock_mod - import litellm.proxy - - if not hasattr(litellm.proxy, "proxy_server"): - litellm.proxy.proxy_server = sys.modules[key] - - -_ensure_proxy_server_mock() - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( pass_through_request, ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py index 8e973fc3771..961479c0393 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py @@ -2,6 +2,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _base_vertex_proxy_route, ) @@ -323,6 +324,7 @@ async def test_vertex_passthrough_forwards_anthropic_beta_header(): vertex_location="us-central1", base_target_url="https://us-central1-aiplatform.googleapis.com", get_vertex_pass_through_handler=mock_handler, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-secret-key"), ) # Verify that allowlisted headers are preserved @@ -417,6 +419,7 @@ async def test_vertex_passthrough_does_not_forward_litellm_auth_token(): vertex_location="us-central1", base_target_url="https://us-central1-aiplatform.googleapis.com", get_vertex_pass_through_handler=mock_handler, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-litellm-secret-key"), ) # The ONLY Authorization header should be the Vertex token