diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 270a1cc2f3b..df784bf6ef4 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -11,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.secret_managers.main import ( + GOOGLE_OIDC_SECRET_PREFIX, UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference, @@ -46,6 +47,14 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | def _resolve_oidc_reference(credential: str | None) -> str | None: if credential is None: return None + if credential.startswith(GOOGLE_OIDC_SECRET_PREFIX): + raise HTTPException( + status_code=401, + detail=( + "pass-through credential oidc/google secret reference could not be resolved: pass-through routes " + "do not fetch Google ID tokens. Use a static key, oidc/file/ or oidc/env/." + ), + ) match resolve_local_oidc_reference(credential): case UnresolvableSecretReference(kind=kind, reason=reason): raise HTTPException( diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 03434115cd8..02cf53d00ec 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -228,7 +228,9 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX): return resolve_local_oidc_reference(value) try: - token: Final = get_secret_str(value) + token: Final = _google_id_token( + value, value.removeprefix(GOOGLE_OIDC_SECRET_PREFIX), timeout=_google_metadata_timeout() + ) except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: return _logged_unresolvable( value, @@ -244,7 +246,7 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool: match exc: - case httpx.TimeoutException() | httpx.NetworkError(): + case httpx.TransportError(): return True case GoogleMetadataStatusError(status_code=status_code): return status_code >= 500 @@ -252,6 +254,38 @@ def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | return False +def _google_id_token(reference: str, audience: str, timeout: float | None) -> str: + cached: Final = cached_oidc_token(reference) + if cached is not None: + return cached + oidc_client: Final = _get_oidc_http_handler() + try: + # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature + response: Final = oidc_client.get( + _google_metadata_identity_url(), + params={"audience": audience}, + headers={"Metadata-Flavor": "Google"}, + timeout=timeout, + ) + finally: + oidc_client.close() + if response.status_code != 200: + raise GoogleMetadataStatusError(response.status_code) + token: Final = response.text + ttl: Final = _oidc_token_cache_ttl(token, 3600 - 60) + if not token: + verbose_logger.warning("Google OIDC token for %s is empty; not caching it", _single_line(reference)) + elif ttl > 0: + oidc_cache.set_cache(key=reference, value=token, ttl=ttl) + else: + verbose_logger.warning( + "Google OIDC token for %s is already expired or expires within %ss; not caching it", + _single_line(reference), + _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS, + ) + return token + + def cached_oidc_token(reference: str) -> str | None: cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped return cached if isinstance(cached, str) and cached else None @@ -340,36 +374,7 @@ def get_secret( oidc_aud = "/".join(secret_name_split.split("/")[1:]) # TODO: Add caching for HTTP requests if oidc_provider == "google": - oidc_token = oidc_cache.get_cache(key=secret_name) - if oidc_token is not None: - return oidc_token - - oidc_client = _get_oidc_http_handler() - # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature - response = oidc_client.get( - _google_metadata_identity_url(), - params={"audience": oidc_aud}, - headers={"Metadata-Flavor": "Google"}, - timeout=_google_metadata_timeout(), - ) - if response.status_code == 200: - oidc_token = response.text - ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60) - if not oidc_token: - verbose_logger.warning( - "Google OIDC token for %s is empty; not caching it", _single_line(secret_name) - ) - elif ttl > 0: - oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl) - else: - verbose_logger.warning( - "Google OIDC token for %s is already expired or expires within %ss; not caching it", - _single_line(secret_name), - _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS, - ) - return oidc_token - else: - raise GoogleMetadataStatusError(response.status_code) + return _google_id_token(secret_name, oidc_aud, timeout=None) elif oidc_provider == "circleci": # https://circleci.com/docs/openid-connect-tokens/ env_secret = os.getenv("CIRCLE_OIDC_TOKEN") diff --git a/litellm/utils.py b/litellm/utils.py index 913a42e513d..f64a88d67b4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -773,14 +773,6 @@ def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, objec ) -def resolve_api_key_secret_reference( - api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] -) -> str | None: - if not isinstance(api_key, AdminApiKeySecretReference): - return api_key - return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream) - - async def aresolve_api_key_secret_reference( api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] ) -> str | None: @@ -802,10 +794,40 @@ async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolv case str() as token: return resolve(lambda _reference: token) case None: - caller_context: Final = contextvars.copy_context() - return await asyncio.get_running_loop().run_in_executor( - _OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference) + _raise_if_the_reference_may_not_be_used(resolve) + fetched: Final = await asyncio.shield( + _google_token_fetch(asyncio.get_running_loop(), api_key).start_or_join() ) + return resolve(lambda _reference: fetched) + + +def _raise_if_the_reference_may_not_be_used(resolve: Callable[[OidcResolver], object]) -> None: + _ = resolve(lambda _reference: "") + + +@dataclass(slots=True) +class _GoogleTokenFetch: + reference: str + in_flight: asyncio.Future[str | UnresolvableSecretReference] | None = None + + def start_or_join(self) -> asyncio.Future[str | UnresolvableSecretReference]: + if self.in_flight is not None: + return self.in_flight + caller_context: Final = contextvars.copy_context() + fetch: Final = asyncio.get_running_loop().run_in_executor( + _OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve_api_key_oidc_reference, self.reference) + ) + fetch.add_done_callback(self._land) + self.in_flight = fetch + return fetch + + def _land(self, _fetch: asyncio.Future[str | UnresolvableSecretReference]) -> None: + self.in_flight = None + + +@lru_cache(maxsize=256) +def _google_token_fetch(loop: asyncio.AbstractEventLoop, reference: str) -> _GoogleTokenFetch: + return _GoogleTokenFetch(reference) def _secret_behind_api_key_reference( diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 9c50a6d4df6..ab4f3f7a675 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -485,7 +485,7 @@ def test_oidc_env_path_reference_is_not_resolved(tmp_path, monkeypatch): ) -def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch): +def test_oidc_google_reference_is_refused_without_a_metadata_fetch(monkeypatch): def fail(*args, **kwargs): raise AssertionError("network-backed oidc reference was resolved inside the request path") @@ -494,11 +494,24 @@ def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch): lambda name, *a, **k: "oidc/google/https://example.com" if name == "OPENAI_API_KEY" else fail(), ) - assert ( + with pytest.raises(HTTPException) as excinfo: _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) - == "oidc/google/https://example.com" + + assert excinfo.value.status_code == 401 + assert "pass-through credential oidc/google" in str(excinfo.value.detail) + assert "example.com" not in str(excinfo.value.detail) + + +def test_deployment_oidc_google_reference_is_not_sent_upstream(monkeypatch): + llm_router = litellm.Router( + model_list=[_flagged_deployment("openai/gpt-4o", api_key="oidc/google/https://vllm.example.test")] ) + with pytest.raises(HTTPException) as excinfo: + _passthrough_router(llm_router).get_credentials(custom_llm_provider="openai", region_name=None) + + assert excinfo.value.status_code == 401 + @pytest.fixture() async def _drain_logging_worker(): diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index c3aa6ca8dd1..a2e5b498f34 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -51,6 +51,9 @@ class MockHTTPHandler: mock_response.json.return_value = self.json_data return mock_response + def close(self): + pass + @pytest.fixture def mock_oidc_cache(): @@ -248,6 +251,24 @@ def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_ assert token == "metadata-host-token" +def test_oidc_google_secret_outside_a_model_api_key_keeps_its_metadata_wait_when_gce_metadata_timeout_is_set( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1") + audience: Final = f"https://sts.example.test/{uuid.uuid4().hex}" + + with respx.mock(assert_all_called=True) as metadata: + identity: Final = metadata.get( + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", + params={"audience": audience}, + ).respond(200, text="web-identity-token") + token: Final = get_secret(f"oidc/google/{audience}") + waits: Final = identity.calls.last.request.extensions["timeout"] + + assert token == "web-identity-token" + assert waits == {"connect": 5.0, "read": 600.0, "write": 600.0, "pool": 600.0} + + def test_oidc_circleci_success(monkeypatch): monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f91978cfe93..f9085b30908 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -7518,8 +7518,9 @@ async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_an (lambda: httpx.Response(200, text=""), 401), (lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503), (lambda: httpx.ReadTimeout("timed out"), 503), + (lambda: httpx.RemoteProtocolError("Server disconnected without sending a response."), 503), ], - ids=["server error", "refused", "empty token", "not on GCP", "timeout"], + ids=["server error", "refused", "empty token", "not on GCP", "timeout", "disconnected"], ) async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call( monkeypatch: pytest.MonkeyPatch, @@ -7610,6 +7611,71 @@ async def test_a_pending_oidc_google_metadata_fetch_leaves_the_default_executor_ assert chat.call_count == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metadata_answer", + [lambda token: httpx.Response(200, text=token), lambda _token: httpx.Response(500, text="metadata server down")], + ids=["token", "outage"], +) +async def test_concurrent_calls_that_miss_the_oidc_google_token_cache_share_one_metadata_fetch( + monkeypatch: pytest.MonkeyPatch, metadata_answer: Callable[[str], httpx.Response] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(time, "time", lambda: _CLOCK_START) + audience, reference = _google_reference() + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + token: Final = _google_id_token(audience, _CLOCK_START) + fetch_started: Final[queue.SimpleQueue[int]] = queue.SimpleQueue() + release: Final = threading.Event() + + def held_metadata(_request: httpx.Request) -> httpx.Response: + fetch_started.put(threading.get_ident()) + release.wait(timeout=30) + return metadata_answer(token) + + with ThreadPoolExecutor(max_workers=1) as observer, respx.mock(assert_all_called=False) as upstream: + metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=held_metadata) + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + calls: Final = [asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa")) for _ in range(24)] + try: + await asyncio.wait_for(asyncio.get_running_loop().run_in_executor(observer, fetch_started.get), timeout=30) + await asyncio.sleep(0.5) + finally: + release.set() + outcomes: Final = await asyncio.gather(*calls, return_exceptions=True) + sent: Final = [call.request.headers["Authorization"] for call in chat.calls] + + assert metadata.call_count == 1 + if metadata.calls.last.response.status_code == 200: + assert outcomes == [None] * 24 + assert sent == [f"Bearer {token}"] * 24 + else: + assert [getattr(outcome, "status_code", None) for outcome in outcomes] == [503] * 24 + assert sent == [] + + +@pytest.mark.asyncio +async def test_a_call_an_oidc_google_api_key_may_not_go_to_is_refused_without_a_metadata_fetch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, reference = _google_reference() + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + + with respx.mock(assert_all_called=False) as upstream: + metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).respond(500) + with pytest.raises(litellm.AuthenticationError) as excinfo: + await acredential_updates( + { + "model": _VLLM_MODEL, + "litellm_credential_name": "vllm-sa", + "api_base": "https://elsewhere.example.test/v1", + } + ) + + assert "only to that credential's own api_base" in excinfo.value.message + assert metadata.call_count == 0 + + @pytest.mark.parametrize( ("configured", "bound"), [(None, 3.0), ("1.5", 1.5), ("three seconds", 3.0), ("0", 3.0), ("-2", 3.0)],