From 01e023d4e97348fe40dbe5f9005a73ad4057fde4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:37:56 -0700 Subject: [PATCH 1/4] test(secret_managers): cover oidc/google api_key references resolved per request Unit tests send a metadata server ID token for an admin oidc/google/ api_key, reuse it within its lifetime, refresh it before it expires, fetch it off the event loop, fail closed with a 401 that names only the kind, and honor GCE_METADATA_HOST. The integration file scripts a metadata server and checks the bearer the upstream receives for config, stored credential and DB credential sources, plus a metadata failure --- .../security/test_api_key_secret_reference.py | 148 ++++++++++++++- .../test_secret_managers_main.py | 19 ++ tests/unit/test_utils.py | 168 +++++++++++++++++- 3 files changed, 327 insertions(+), 8 deletions(-) diff --git a/tests/integration/security/test_api_key_secret_reference.py b/tests/integration/security/test_api_key_secret_reference.py index 0b1187d2e0e..4f69845a046 100644 --- a/tests/integration/security/test_api_key_secret_reference.py +++ b/tests/integration/security/test_api_key_secret_reference.py @@ -1,6 +1,8 @@ from __future__ import annotations +import base64 import json +import time import uuid from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass @@ -46,7 +48,17 @@ _SEARCH_PATH: Final = f"/v1/vector_stores/{_VECTOR_STORE}/search" _BATCH_DEPLOYMENT: Final = "wif-vllm-batch" _BEDROCK_MODEL: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" _PROVIDER_INPUT_TOKENS: Final = 7919 -_UNSUPPORTED_REFERENCE: Final = "oidc/google/https://example.invalid" +_UNSUPPORTED_REFERENCE: Final = "oidc/github/https://example.invalid" +_GOOGLE_MODEL: Final = "hosted_vllm/wif-google" +_GOOGLE_AUDIENCES: Final = MappingProxyType( + { + "config_inline": "https://vllm-inline.example.test", + "config_credential": "https://vllm-credential.example.test", + "db_credential": "https://vllm-db-credential.example.test", + } +) +_UNAVAILABLE_AUDIENCE: Final = "https://vllm-unavailable.example.test" +_IDENTITY_PATH: Final = "/computeMetadata/v1/instance/service-accounts/default/identity" _RUN_SECRETS_TOKEN: Final = "/run/secrets/wif-absent-token" _SOURCES: Final = ("config_credential", "config_inline", "db_credential", "db_inline") _UNREADABLE: Final = MappingProxyType( @@ -108,6 +120,43 @@ def _reference(path: Path) -> str: return f"oidc/file/{path}" +def _google_reference(audience: str) -> str: + return f"oidc/google/{audience}" + + +def _jwt_segment(part: Mapping[str, JsonValue]) -> str: + return base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=") + + +@dataclass(frozen=True, slots=True) +class _Google: + expires: int + + def token(self, audience: str) -> str: + claims: Final = _jwt_segment({"iss": "https://accounts.google.com", "aud": audience, "exp": self.expires}) + return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{claims}.signature" + + def bearers(self) -> frozenset[str]: + return frozenset(f"Bearer {self.token(audience)}" for audience in _GOOGLE_AUDIENCES.values()) + + +def _metadata_server(google: _Google) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + target: Final = httpx.URL(request.target) + audience: Final = target.params.get("audience") + if audience is None or (request.method, target.path, request.headers.get("metadata-flavor")) != ( + "GET", + _IDENTITY_PATH, + "Google", + ): + return Reply(status=404, body=b"not found", content_type="text/plain") + if audience == _UNAVAILABLE_AUDIENCE: + return Reply(status=503, body=b"metadata server unavailable", content_type="text/plain") + return Reply(body=google.token(audience).encode(), content_type="text/plain") + + return respond + + def _secrets(directory: Path) -> _Secrets: allowed: Final = directory / "secrets" outside_dir: Final = directory / "outside" @@ -169,9 +218,9 @@ def _served(request: Request) -> Reply: return openai_error(404) -def _provider(secrets: _Secrets) -> Callable[[Request], Reply]: +def _provider(secrets: _Secrets, google: _Google) -> Callable[[Request], Reply]: def respond(request: Request) -> Reply: - if request.headers.get("authorization") != f"Bearer {secrets.current()}": + if request.headers.get("authorization") not in {f"Bearer {secrets.current()}", *google.bearers()}: return openai_error(401) return _served(request) @@ -224,6 +273,19 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path: "aws_bedrock_runtime_endpoint": provider.removesuffix("/v1"), }, ), + _deployment( + "wif-google-inline", + { + "model": _GOOGLE_MODEL, + "api_key": _google_reference(_GOOGLE_AUDIENCES["config_inline"]), + "api_base": provider, + }, + ), + _deployment("wif-google-credential", _named(_GOOGLE_MODEL, "wif-google")), + _deployment( + "wif-google-unavailable", + {"model": _GOOGLE_MODEL, "api_key": _google_reference(_UNAVAILABLE_AUDIENCE), "api_base": provider}, + ), _deployment( "wif-mistral-passthrough", { @@ -239,6 +301,7 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path: _credential("wif-outside", _reference(secrets.outside), provider), _credential("wif-unsupported", _UNSUPPORTED_REFERENCE, provider), _credential("wif-run-secrets", f"oidc/file/{_RUN_SECRETS_TOKEN}", provider), + _credential("wif-google", _google_reference(_GOOGLE_AUDIENCES["config_credential"]), provider), ], "vector_store_registry": [ { @@ -312,6 +375,8 @@ class _Rig: owned: OwnedProxy provider: Recorder intruder: Recorder + metadata: Recorder + google: _Google provider_base: str intruder_base: str secrets: _Secrets @@ -339,17 +404,23 @@ def _is_discovery(request: Request) -> bool: def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: directory: Final = tmp_path_factory.mktemp("api_key_secret_reference").resolve() secrets: Final = _secrets(directory) + google: Final = _Google(expires=int(time.time()) + 3600) with ( gateway_from_environment() as gateway, - wire_server(_provider(secrets)) as provider_wire, + wire_server(_provider(secrets, google)) as provider_wire, wire_server(_intruder) as intruder_wire, + wire_server(_metadata_server(google)) as metadata_wire, ): provider: Final = Recorder(provider_wire) provider_base: Final = f"{provider_wire.url}/v1" with owned_proxy_process( gateway, directory, - {**secrets.overrides(), "MISTRAL_API_BASE": provider_wire.url}, + { + **secrets.overrides(), + "MISTRAL_API_BASE": provider_wire.url, + "GCE_METADATA_HOST": httpx.URL(metadata_wire.url).netloc.decode(), + }, config=_config(directory, secrets, provider_base), workers=2, healthcheck=WorkerHealthcheck.LENIENT, @@ -361,6 +432,8 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: owned=owned, provider=provider, intruder=Recorder(intruder_wire), + metadata=Recorder(metadata_wire), + google=google, provider_base=provider_base, intruder_base=f"{intruder_wire.url}/v1", secrets=secrets, @@ -708,7 +781,70 @@ def test_team_row_naming_the_credential_sends_the_token_only_to_its_host(rig: _R def test_startup_discovery_sends_the_boot_token(rig: _Rig) -> None: sent: Final = frozenset(_bearers(tuple(request for request in rig.boot if _is_discovery(request)))) - assert sent == {f"Bearer {rig.boot_token}"}, sent + assert sent - rig.google.bearers() == {f"Bearer {rig.boot_token}"}, sent + + +def _discovery_bearers(rig: _Rig) -> frozenset[str | None]: + return frozenset(_bearers(tuple(request for request in rig.provider.requests() if _is_discovery(request)))) + + +def test_startup_discovery_sends_the_google_token(rig: _Rig) -> None: + expected: Final = frozenset( + f"Bearer {rig.google.token(_GOOGLE_AUDIENCES[source])}" for source in ("config_inline", "config_credential") + ) + sent: Final = eventually( + partial(_discovery_bearers, rig), lambda seen: expected <= seen, seconds=30, return_last_on_timeout=True + ) + assert expected <= sent, sent + + +def _google_model(rig: _Rig, scenario: Scenario, source: str) -> str: + match source: + case "config_inline": + return "wif-google-inline" + case "config_credential": + return "wif-google-credential" + case "db_credential": + credential: Final = f"db-google-{uuid.uuid4().hex}" + rig.owned.gateway.post( + "/credentials", + _credential(credential, _google_reference(_GOOGLE_AUDIENCES[source]), rig.provider_base), + ) + scenario.cleanups.callback(_delete_credential, rig.owned.gateway, credential) + name, identity = _created_model( + rig.owned.gateway, + scenario, + _deployment(f"db-google-{uuid.uuid4().hex}", _named(_GOOGLE_MODEL, credential)), + ) + eventually(partial(rig.owned.serves_only_on_every_worker, name, identity), bool, seconds=120) + return name + case _: + pytest.fail(f"unknown source {source}") + + +def _asked_audiences(rig: _Rig) -> frozenset[str | None]: + return frozenset(httpx.URL(request.target).params.get("audience") for request in rig.metadata.requests()) + + +@pytest.mark.parametrize("source", tuple(_GOOGLE_AUDIENCES)) +def test_google_reference_sends_the_metadata_server_token_for_its_audience(rig: _Rig, source: str) -> None: + audience: Final = _GOOGLE_AUDIENCES[source] + with rig.owned.gateway.scenario() as scenario: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, _google_model(rig, scenario, source), marker) + sent: Final = _bearers(_carrying(rig.provider.requests(), marker)) + assert (response.status_code, sent) == (200, (f"Bearer {rig.google.token(audience)}",)), response.text + assert audience in _asked_audiences(rig) + + +def test_google_reference_fails_closed_when_the_metadata_server_fails(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _chat(rig.owned.gateway, "wif-google-unavailable", marker) + reached: Final = _carrying(rig.provider.requests(), marker) + assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert "oidc/google secret reference could not be resolved" in response.text, response.text + assert _UNAVAILABLE_AUDIENCE.removeprefix("https://") not in response.text, response.text + assert _UNAVAILABLE_AUDIENCE in _asked_audiences(rig) def _side_call(rig: _Rig, seam: str, marker: str) -> tuple[httpx.Response, Callable[[Request], bool]]: diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index 7c6e1de6949..b3948efeac8 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -3,10 +3,12 @@ import json import logging import os import time +import uuid from typing import Final from unittest.mock import Mock, patch import pytest +import respx import litellm from litellm._logging import redact_internal_details_from_client_message @@ -224,6 +226,23 @@ def test_oidc_google_failure(): get_secret(secret_name) + +def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GCE_METADATA_HOST", "127.0.0.1:8930") + audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}" + + with respx.mock(assert_all_called=True) as metadata: + metadata.get( + "http://127.0.0.1:8930/computeMetadata/v1/instance/service-accounts/default/identity", + params={"audience": audience}, + headers={"Metadata-Flavor": "Google"}, + ).respond(200, text="metadata-host-token") + token: Final = get_secret(f"oidc/google/{audience}") + + assert token == "metadata-host-token" + def test_oidc_circleci_success(monkeypatch): monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9b6700e9d39..1ec59cfa2a9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10,9 +10,12 @@ import logging import os import queue import threading +import time +import uuid from collections.abc import Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone +from functools import partial from pathlib import Path, PurePath from typing import Final, SupportsIndex, cast from unittest import mock @@ -7165,7 +7168,6 @@ def test_unresolvable_oidc_api_key_fails_closed_naming_the_reason_without_its_ta ("reference", "kind"), [ ("oidc/env_path/MISTRAL_SA_TOKEN_PATH", "oidc/env_path"), - ("oidc/google/https://api.mistral.ai", "oidc/google"), ("oidc/github/mistral", "oidc/github"), ("oidc/env_MISTRAL_SA_TOKEN_PATH", "oidc/"), ("oidc/LIT-UNKNOWN-KIND/MISTRAL_SA_TOKEN_PATH", "oidc/"), @@ -7186,6 +7188,7 @@ def test_oidc_api_key_of_a_kind_api_key_cannot_resolve_fails_closed_as_an_authen assert not route.called assert excinfo.value.status_code == 401 assert f"{kind} secret reference" in excinfo.value.message + assert "oidc/google/" in excinfo.value.message assert reference.removeprefix(kind).lstrip("/") not in excinfo.value.message assert "MISTRAL_SA_TOKEN_PATH" not in excinfo.value.message @@ -7201,7 +7204,7 @@ async def test_router_fails_over_from_a_deployment_whose_oidc_api_key_kind_canno "model_name": "devstral", "litellm_params": { "model": "mistral/devstral-latest", - "api_key": "oidc/google/https://api.mistral.ai", + "api_key": "oidc/github/mistral", "order": 1, }, }, @@ -7240,6 +7243,167 @@ def test_an_oidc_api_key_sent_with_the_call_goes_upstream_as_written( assert sent == f"Bearer {reference}" +_GOOGLE_METADATA_IDENTITY: Final = ( + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity" +) +_VLLM_BASE: Final = "https://vllm.example.test/v1" +_VLLM_CHAT_URL: Final = f"{_VLLM_BASE}/chat/completions" +_VLLM_MODEL: Final = "hosted_vllm/qwen3-0.6b" +_VLLM_REPLY: Final = {**_MISTRAL_REPLY, "model": "qwen3-0.6b"} +_CLOCK_START: Final = 1_900_000_000.0 +_GOOGLE_TOKEN_LIFETIME: Final = 3600 + + +def _jwt_segment(claims: Mapping[str, object]) -> str: + return base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + + +def _google_id_token(audience: str, issued_at: float) -> str: + claims: Final = {"aud": audience, "iat": int(issued_at), "exp": int(issued_at) + _GOOGLE_TOKEN_LIFETIME} + return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{_jwt_segment(claims)}.signature" + + +def _google_reference() -> tuple[str, str]: + audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}" + return audience, f"oidc/google/{audience}" + + +def _metadata_identity(fetched_on: queue.SimpleQueue[int], request: httpx.Request) -> httpx.Response: + fetched_on.put(threading.get_ident()) + if request.headers.get("Metadata-Flavor") != "Google": + return httpx.Response(403) + return httpx.Response(200, text=_google_id_token(request.url.params["audience"], time.time())) + + +def _vllm_credential(api_key: str) -> CredentialItem: + return CredentialItem( + credential_name="vllm-sa", credential_values={"api_key": api_key, "api_base": _VLLM_BASE}, credential_info={} + ) + + +async def _ping_vllm_through(call_path: str, credential_name: str) -> None: + if call_path == "async": + await litellm.acompletion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name) + return + litellm.completion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_path", ["sync", "async"]) +async def test_named_credential_oidc_google_api_key_sends_the_metadata_token_and_refreshes_it_before_it_expires( + monkeypatch: pytest.MonkeyPatch, call_path: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(time, "time", lambda: _CLOCK_START) + audience, reference = _google_reference() + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + refreshed_at: Final = _CLOCK_START + _GOOGLE_TOKEN_LIFETIME - 59 + + with respx.mock(assert_all_called=True) as upstream: + metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock( + side_effect=partial(_metadata_identity, queue.SimpleQueue()) + ) + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + await _ping_vllm_through(call_path, "vllm-sa") + await _ping_vllm_through(call_path, "vllm-sa") + fetches_within_lifetime: Final = metadata.call_count + monkeypatch.setattr(time, "time", lambda: refreshed_at) + await _ping_vllm_through(call_path, "vllm-sa") + sent: Final = [call.request.headers["Authorization"] for call in chat.calls] + audiences: Final = [call.request.url.params["audience"] for call in metadata.calls] + + first_token: Final = f"Bearer {_google_id_token(audience, _CLOCK_START)}" + assert sent == [first_token, first_token, f"Bearer {_google_id_token(audience, refreshed_at)}"] + assert (fetches_within_lifetime, audiences) == (1, [audience, audience]) + assert litellm.credential_list[0].credential_values["api_key"] == reference + + +@pytest.mark.asyncio +async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_and_keeps_the_reference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(time, "time", lambda: _CLOCK_START) + audience, reference = _google_reference() + fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vllm", + "litellm_params": {"model": _VLLM_MODEL, "api_base": _VLLM_BASE, "api_key": reference}, + } + ] + ) + + try: + with respx.mock(assert_all_called=True) as upstream: + upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on)) + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + await router.acompletion(model="vllm", messages=_PING) + sent: Final = chat.calls.last.request.headers["Authorization"] + finally: + router.discard() + + assert sent == f"Bearer {_google_id_token(audience, _CLOCK_START)}" + assert fetched_on.qsize() == 1 + assert fetched_on.get_nowait() != threading.get_ident() + assert router.get_model_list(model_name="vllm")[0]["litellm_params"]["api_key"] == reference + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_path", ["sync", "async"]) +@pytest.mark.parametrize( + "metadata_failure", + [ + lambda: httpx.Response(500, text="metadata server unavailable"), + lambda: httpx.Response(200, text=""), + lambda: httpx.ConnectError("[Errno -2] Name or service not known"), + lambda: httpx.ReadTimeout("timed out"), + ], + ids=["server error", "empty token", "not on GCP", "timeout"], +) +async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + call_path: str, + metadata_failure: Callable[[], httpx.Response | httpx.HTTPError], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(time, "time", lambda: _CLOCK_START) + audience, reference = _google_reference() + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + recovered_token: Final = _google_id_token(audience, _CLOCK_START) + + with respx.mock(assert_all_called=True) as upstream: + upstream.get(_GOOGLE_METADATA_IDENTITY).mock( + side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)] + ) + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(litellm.AuthenticationError) as excinfo: + await _ping_vllm_through(call_path, "vllm-sa") + sent_while_failing: Final = chat.call_count + await _ping_vllm_through(call_path, "vllm-sa") + sent_after_recovery: Final = [call.request.headers["Authorization"] for call in chat.calls] + + assert excinfo.value.status_code == 401 + assert "api_key oidc/google secret reference could not be resolved" in excinfo.value.message + assert audience.removeprefix("https://") not in excinfo.value.message + assert reference in caplog.text + assert sent_while_failing == 0 + assert sent_after_recovery == [f"Bearer {recovered_token}"] + + +def test_an_oidc_google_api_key_sent_with_the_call_goes_upstream_as_written_without_a_metadata_fetch() -> None: + _, reference = _google_reference() + + with respx.mock(assert_all_called=True) as upstream: + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + litellm.completion(model=_VLLM_MODEL, api_base=_VLLM_BASE, messages=_PING, api_key=reference) + sent: Final = chat.calls.last.request.headers["Authorization"] + + assert sent == f"Bearer {reference}" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model_info", "expected"), From ee1fa39ca6cb93651aca19df619153ac9aa5a06f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:59:07 -0700 Subject: [PATCH 2/4] feat(secret_managers): resolve oidc/google api_key references per request An admin-set oidc/google/ api_key now resolves on each call to the GCE metadata server's ID token for that audience, cached until a minute before it expires. Async callers fetch on a cache miss in a worker thread, so the event loop never waits on the metadata server. A failed fetch answers 401 naming only the kind, and GCE_METADATA_HOST points the fetch at another metadata server. --- .../litellm_executed_batches.py | 8 ++-- litellm/proxy/proxy_server.py | 15 +++--- litellm/realtime_api/main.py | 4 +- litellm/router.py | 4 +- litellm/secret_managers/main.py | 47 +++++++++++++++++-- litellm/utils.py | 45 ++++++++++++++---- 6 files changed, 92 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/batches_endpoints/litellm_executed_batches.py b/litellm/proxy/batches_endpoints/litellm_executed_batches.py index 098dd2f9816..67728e0fd63 100644 --- a/litellm/proxy/batches_endpoints/litellm_executed_batches.py +++ b/litellm/proxy/batches_endpoints/litellm_executed_batches.py @@ -35,7 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders -from litellm.utils import resolve_api_key_secret_reference +from litellm.utils import aresolve_api_key_secret_reference if TYPE_CHECKING: from prisma import types as prisma_types @@ -227,7 +227,7 @@ async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_clie return response.status_code == httpx.codes.NOT_FOUND -def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None: +async def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None: model: Final = credentials.get("model") api_base: Final = credentials.get("api_base") api_key: Final = credentials.get("api_key") @@ -244,7 +244,7 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, return None if resolved_api_base is None: return None - return resolved_api_base, resolve_api_key_secret_reference( + return resolved_api_base, await aresolve_api_key_secret_reference( resolved_api_key, model=model, llm_provider=provider, upstream=credentials ) @@ -255,7 +255,7 @@ async def litellm_executed_provider_for( provider: Final = litellm_executed_provider_of(credentials) if provider is None: return None - upstream: Final = _upstream_of(credentials, provider) + upstream: Final = await _upstream_of(credentials, provider) if upstream is None: return None return provider if await lacks_files_api(*upstream) else None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 967cae27990..9b5b8faa164 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -201,7 +201,7 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential +from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential if TYPE_CHECKING: from aiohttp import ClientSession @@ -14651,10 +14651,10 @@ def _provider_count_outcome( return result -def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: +async def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params) try: - load_credentials_from_list(call_params) + return {**call_params, **(await acredential_updates(call_params))} except litellm.AuthenticationError as e: raise ProxyException( message=e.message, @@ -14662,13 +14662,12 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]: param="model", code=401, ) from e - return call_params -def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None: +async def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None: if deployment is None: return None - return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))} + return {**deployment, "litellm_params": await _with_resolved_api_key(deployment.get("litellm_params", {}))} _DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -14731,7 +14730,7 @@ async def _try_provider_token_count( model_to_use=model_to_use or "", messages=messages, contents=contents, - deployment=_with_resolved_credentials(deployment_in_region), + deployment=await _with_resolved_credentials(deployment_in_region), request_model=request_model, tools=tools, system=system, @@ -14765,7 +14764,7 @@ async def _try_native_responses_count( model_to_use=model_to_use, request=responses_request, policy=policy, - deployment=_with_resolved_credentials(deployment), + deployment=await _with_resolved_credentials(deployment), request_model=request_model, ) ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 088dfb54155..e33dcf72781 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ProviderConfigManager, aresolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -802,7 +802,7 @@ async def realtime_health_check( resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints str | None, api_base or resolved_params.get("api_base") ) - resolved_api_key: Final = resolve_api_key_secret_reference( + resolved_api_key: Final = await aresolve_api_key_secret_reference( cast( # cast-ok: provider parameters expose optional string credentials str | None, api_key or resolved_params.get("api_key") ), diff --git a/litellm/router.py b/litellm/router.py index a18ad7c3a97..d60b1d36505 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -437,6 +437,7 @@ from litellm.utils import ( ModelResponse, Rules, TextCompletionResponse, + aresolve_api_key_secret_reference, function_setup, get_llm_provider, get_non_default_completion_params, @@ -447,7 +448,6 @@ from litellm.utils import ( lookup_model_info_for_registration, model_info_with_pricing_overrides, provider_rejectable_params, - resolve_api_key_secret_reference, set_live_deployment_replay, ) @@ -11632,7 +11632,7 @@ class Router: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): return - api_key: Final = resolve_api_key_secret_reference( + api_key: Final = await aresolve_api_key_secret_reference( params.api_key or dynamic_api_key, model=model, llm_provider=provider, diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index f4234e26c60..ee76b30c585 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -166,6 +166,15 @@ def get_secret_str( LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/") +GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/" +API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX) +_GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal" +_GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token" + + +def _google_metadata_identity_url() -> str: + host: Final = os.getenv("GCE_METADATA_HOST") or _GOOGLE_METADATA_DEFAULT_HOST + return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity" @dataclass(frozen=True, slots=True) @@ -197,13 +206,39 @@ def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReferenc return secret -def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference: +def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretReference: + if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX): + return resolve_local_oidc_reference(value) + try: + token: Final = get_secret_str(value) + except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: + return _logged_unresolvable(value, "oidc/google", _GOOGLE_TOKEN_UNAVAILABLE, f"{type(exc).__name__}: {exc}") + if not token: + return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value") + return token + + +def google_oidc_token_needs_fetch(value: str) -> bool: + return ( + value.startswith(GOOGLE_OIDC_SECRET_PREFIX) and oidc_cache.get_cache(key=value) is None # pyright: ignore[reportUnknownMemberType] # DualCache.get_cache is untyped + ) + + +def _logged_unresolvable( + reference: str, kind: str, reason: str, detail: str | None = None +) -> UnresolvableSecretReference: verbose_logger.warning( - "Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason + "Secret reference %s could not be resolved: %s", + _single_line(reference), + reason if detail is None else f"{reason} ({_single_line(detail)})", ) return UnresolvableSecretReference(kind=kind, reason=reason) +def _single_line(text: str) -> str: + return text.replace("\r", "").replace("\n", "") + + def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str: match exc: case UnicodeDecodeError(): @@ -279,14 +314,16 @@ def get_secret( oidc_client = _get_oidc_http_handler() # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature response = oidc_client.get( - "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", + _google_metadata_identity_url(), params={"audience": oidc_aud}, headers={"Metadata-Flavor": "Google"}, ) if response.status_code == 200: oidc_token = response.text ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60) - if ttl > 0: + if not oidc_token: + verbose_logger.warning("Google OIDC token for %s is empty; not caching it", secret_name) + elif ttl > 0: oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl) else: verbose_logger.warning( @@ -296,7 +333,7 @@ def get_secret( ) return oidc_token else: - raise ValueError("Google OIDC provider failed") + raise ValueError(f"Google OIDC provider failed with HTTP {response.status_code}") elif oidc_provider == "circleci": # https://circleci.com/docs/openid-connect-tokens/ env_secret = os.getenv("CIRCLE_OIDC_TOKEN") diff --git a/litellm/utils.py b/litellm/utils.py index 897138a940c..4adead27388 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -30,7 +30,7 @@ import time import traceback from contextlib import nullcontext from dataclasses import dataclass, field -from functools import lru_cache, wraps +from functools import lru_cache, partial, wraps from importlib import resources from inspect import iscoroutine from io import StringIO @@ -464,11 +464,12 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig from litellm.secret_managers.main import ( - LOCAL_OIDC_SECRET_PREFIXES, + API_KEY_OIDC_SECRET_PREFIXES, UnresolvableSecretReference, get_secret, + google_oidc_token_needs_fetch, oidc_reference_kind, - resolve_local_oidc_reference, + resolve_api_key_oidc_reference, ) from ._logging import is_debugging_on, verbose_logger @@ -765,19 +766,30 @@ def resolve_api_key_secret_reference( return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream) +async def aresolve_api_key_secret_reference( + api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] +) -> str | None: + resolve: Final = partial(resolve_api_key_secret_reference, api_key, model, llm_provider, upstream) + return await asyncio.to_thread(resolve) if _resolving_fetches_a_token(api_key) else resolve() + + +def _resolving_fetches_a_token(api_key: object) -> bool: + return isinstance(api_key, AdminApiKeySecretReference) and google_oidc_token_needs_fetch(api_key) + + def _secret_behind_api_key_reference( reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object] ) -> str: if not reference.may_reach(upstream): raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model) - if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES): + if not reference.startswith(API_KEY_OIDC_SECRET_PREFIXES): raise _secret_reference_error( f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. " - "Use oidc/file/ or oidc/env/.", + "Use oidc/file/, oidc/env/ or oidc/google/.", llm_provider=llm_provider, model=model, ) - match resolve_local_oidc_reference(reference): + match resolve_api_key_oidc_reference(reference): case UnresolvableSecretReference(kind=kind, reason=reason): raise _secret_reference_error( f"api_key {kind} secret reference could not be resolved: {reason}", @@ -814,12 +826,25 @@ def load_credentials_from_list(kwargs: dict[str, object]) -> None: Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key reference for the secret it points to, read now so a rotated token is used on the next call """ - kwargs.update(_credential_updates(kwargs)) + kwargs.update(_credential_updates(kwargs, _named_credential_values(kwargs))) -def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: +async def acredential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: credential_values: Final = _named_credential_values(kwargs) - api_key: Final = kwargs.get("api_key", credential_values.get("api_key")) + resolve: Final = partial(_credential_updates, kwargs, credential_values) + return ( + await asyncio.to_thread(resolve) + if _resolving_fetches_a_token(_api_key_after_credentials(kwargs, credential_values)) + else resolve() + ) + + +def _api_key_after_credentials(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> object: + return kwargs.get("api_key", credential_values.get("api_key")) + + +def _credential_updates(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> Mapping[str, object]: + api_key: Final = _api_key_after_credentials(kwargs, credential_values) if not isinstance(api_key, AdminApiKeySecretReference): return credential_values secret: Final = _secret_behind_api_key_reference( @@ -2192,7 +2217,7 @@ def client(original_function): kwargs["litellm_logging_obj"] = logging_obj ## LOAD CREDENTIALS - load_credentials_from_list(kwargs) + kwargs.update(await acredential_updates(kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is the untyped request dict from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict From 3f557528f40a01c35193cb3dac19cca7509a5891 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:41:41 -0700 Subject: [PATCH 3/4] fix(oidc): bound the Google metadata fetch and answer 503 when it fails A Google OIDC api_key fetch now waits at most GCE_METADATA_TIMEOUT seconds (default 3), runs on its own thread pool instead of the loop's default executor, and sends the token read from the cache at call start even if it expires mid-call. A metadata timeout, network error, or 5xx answers a retryable 503, while a 4xx or empty token stays a 401. The expiry and empty-token warnings log the secret name on one line. --- litellm/secret_managers/main.py | 55 +++++++++++--- litellm/utils.py | 57 +++++++++++---- .../test_secret_managers_main.py | 2 +- tests/unit/test_utils.py | 72 ++++++++++++++++--- 4 files changed, 153 insertions(+), 33 deletions(-) diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index ee76b30c585..5f0c4c17e96 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -6,7 +6,7 @@ import traceback from collections.abc import Callable from dataclasses import dataclass from functools import lru_cache -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError @@ -169,6 +169,7 @@ LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/") GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/" API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX) _GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal" +_GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS: Final = "3" _GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token" @@ -177,10 +178,24 @@ def _google_metadata_identity_url() -> str: return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity" +def _google_metadata_timeout() -> float: + return float(os.getenv("GCE_METADATA_TIMEOUT") or _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS) + + +class GoogleMetadataStatusError(ValueError): + def __init__(self, status_code: int) -> None: + super().__init__(f"Google OIDC provider failed with HTTP {status_code}") + self.status_code: Final = status_code + + @dataclass(frozen=True, slots=True) class UnresolvableSecretReference: kind: str reason: str + retryable: bool = False + + +OidcResolver: TypeAlias = Callable[[str], "str | UnresolvableSecretReference"] _OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"}) @@ -212,27 +227,42 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere try: token: Final = get_secret_str(value) except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: - return _logged_unresolvable(value, "oidc/google", _GOOGLE_TOKEN_UNAVAILABLE, f"{type(exc).__name__}: {exc}") + return _logged_unresolvable( + value, + "oidc/google", + _GOOGLE_TOKEN_UNAVAILABLE, + f"{type(exc).__name__}: {exc}", + retryable=_metadata_server_failed_to_answer(exc), + ) if not token: return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value") return token -def google_oidc_token_needs_fetch(value: str) -> bool: - return ( - value.startswith(GOOGLE_OIDC_SECRET_PREFIX) and oidc_cache.get_cache(key=value) is None # pyright: ignore[reportUnknownMemberType] # DualCache.get_cache is untyped - ) +def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool: + match exc: + case httpx.TimeoutException() | httpx.NetworkError(): + return True + case GoogleMetadataStatusError(status_code=status_code): + return status_code >= 500 + case _: + return False + + +def cached_oidc_token(reference: str) -> str | None: + cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped + return cached if isinstance(cached, str) and cached else None def _logged_unresolvable( - reference: str, kind: str, reason: str, detail: str | None = None + reference: str, kind: str, reason: str, detail: str | None = None, retryable: bool = False ) -> UnresolvableSecretReference: verbose_logger.warning( "Secret reference %s could not be resolved: %s", _single_line(reference), reason if detail is None else f"{reason} ({_single_line(detail)})", ) - return UnresolvableSecretReference(kind=kind, reason=reason) + return UnresolvableSecretReference(kind=kind, reason=reason, retryable=retryable) def _single_line(text: str) -> str: @@ -317,23 +347,26 @@ def get_secret( _google_metadata_identity_url(), params={"audience": oidc_aud}, headers={"Metadata-Flavor": "Google"}, + timeout=_google_metadata_timeout(), ) if response.status_code == 200: oidc_token = response.text ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60) if not oidc_token: - verbose_logger.warning("Google OIDC token for %s is empty; not caching it", secret_name) + verbose_logger.warning( + "Google OIDC token for %s is empty; not caching it", _single_line(secret_name) + ) elif ttl > 0: oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl) else: verbose_logger.warning( "Google OIDC token for %s is already expired or expires within %ss; not caching it", - secret_name, + _single_line(secret_name), _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS, ) return oidc_token else: - raise ValueError(f"Google OIDC provider failed with HTTP {response.status_code}") + raise GoogleMetadataStatusError(response.status_code) elif oidc_provider == "circleci": # https://circleci.com/docs/openid-connect-tokens/ env_secret = os.getenv("CIRCLE_OIDC_TOKEN") diff --git a/litellm/utils.py b/litellm/utils.py index 4adead27388..94c4d552747 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,6 +28,7 @@ import textwrap import threading import time import traceback +from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext from dataclasses import dataclass, field from functools import lru_cache, partial, wraps @@ -465,9 +466,11 @@ from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfi from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig from litellm.secret_managers.main import ( API_KEY_OIDC_SECRET_PREFIXES, + GOOGLE_OIDC_SECRET_PREFIX, + OidcResolver, UnresolvableSecretReference, + cached_oidc_token, get_secret, - google_oidc_token_needs_fetch, oidc_reference_kind, resolve_api_key_oidc_reference, ) @@ -769,16 +772,36 @@ def resolve_api_key_secret_reference( async def aresolve_api_key_secret_reference( api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object] ) -> str | None: - resolve: Final = partial(resolve_api_key_secret_reference, api_key, model, llm_provider, upstream) - return await asyncio.to_thread(resolve) if _resolving_fetches_a_token(api_key) else resolve() + if not isinstance(api_key, AdminApiKeySecretReference): + return api_key + return await _resolved_off_the_event_loop( + partial(_secret_behind_api_key_reference, api_key, model, llm_provider, upstream), api_key + ) -def _resolving_fetches_a_token(api_key: object) -> bool: - return isinstance(api_key, AdminApiKeySecretReference) and google_oidc_token_needs_fetch(api_key) +_OIDC_TOKEN_FETCH_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-oidc-token-fetch") +_Resolved: Final = TypeVar("_Resolved") + + +async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolved], api_key: object) -> _Resolved: + if not isinstance(api_key, AdminApiKeySecretReference) or not api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX): + return resolve(resolve_api_key_oidc_reference) + match cached_oidc_token(api_key): + case str() as token: + return resolve(lambda _reference: token) + case None: + caller_context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + _OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference) + ) def _secret_behind_api_key_reference( - reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object] + reference: AdminApiKeySecretReference, + model: object, + llm_provider: object, + upstream: Mapping[str, object], + resolve_oidc: OidcResolver = resolve_api_key_oidc_reference, ) -> str: if not reference.may_reach(upstream): raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model) @@ -789,7 +812,13 @@ def _secret_behind_api_key_reference( llm_provider=llm_provider, model=model, ) - match resolve_api_key_oidc_reference(reference): + match resolve_oidc(reference): + case UnresolvableSecretReference(kind=kind, reason=reason, retryable=True): + raise ServiceUnavailableError( + message=f"api_key {kind} secret reference could not be resolved: {reason}", + llm_provider=llm_provider, + model=model, + ) case UnresolvableSecretReference(kind=kind, reason=reason): raise _secret_reference_error( f"api_key {kind} secret reference could not be resolved: {reason}", @@ -831,11 +860,8 @@ def load_credentials_from_list(kwargs: dict[str, object]) -> None: async def acredential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]: credential_values: Final = _named_credential_values(kwargs) - resolve: Final = partial(_credential_updates, kwargs, credential_values) - return ( - await asyncio.to_thread(resolve) - if _resolving_fetches_a_token(_api_key_after_credentials(kwargs, credential_values)) - else resolve() + return await _resolved_off_the_event_loop( + partial(_credential_updates, kwargs, credential_values), _api_key_after_credentials(kwargs, credential_values) ) @@ -843,12 +869,17 @@ def _api_key_after_credentials(kwargs: Mapping[str, object], credential_values: return kwargs.get("api_key", credential_values.get("api_key")) -def _credential_updates(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> Mapping[str, object]: +def _credential_updates( + kwargs: Mapping[str, object], + credential_values: Mapping[str, object], + resolve_oidc: OidcResolver = resolve_api_key_oidc_reference, +) -> Mapping[str, object]: api_key: Final = _api_key_after_credentials(kwargs, credential_values) if not isinstance(api_key, AdminApiKeySecretReference): return credential_values secret: Final = _secret_behind_api_key_reference( api_key, + resolve_oidc=resolve_oidc, model=kwargs.get("model"), llm_provider=kwargs.get("custom_llm_provider"), upstream=MappingProxyType( diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index b3948efeac8..19b6bae0462 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -34,7 +34,7 @@ class MockHTTPHandler: self.text = "mocked_token" self.json_data = {"value": "mocked_token"} - def get(self, url, params=None, headers=None): + def get(self, url, params=None, headers=None, timeout=None): # Store params for audience verification self.last_params = params logger.debug( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 1ec59cfa2a9..7f922757e84 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5,10 +5,12 @@ import copy import contextlib import contextvars import io +import itertools import json import logging import os import queue +import socket import threading import time import uuid @@ -82,8 +84,10 @@ from litellm.types.utils import ( all_litellm_params, bedrock_batch_litellm_params, ) +from litellm.secret_managers.main import oidc_cache from litellm.types.videos.main import VideoObject from litellm.utils import ( + acredential_updates, _invalidate_model_cost_lowercase_map, add_custom_logger_callback_to_specific_event, check_valid_key, @@ -7353,20 +7357,22 @@ async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_an @pytest.mark.asyncio @pytest.mark.parametrize("call_path", ["sync", "async"]) @pytest.mark.parametrize( - "metadata_failure", + ("metadata_failure", "status_code"), [ - lambda: httpx.Response(500, text="metadata server unavailable"), - lambda: httpx.Response(200, text=""), - lambda: httpx.ConnectError("[Errno -2] Name or service not known"), - lambda: httpx.ReadTimeout("timed out"), + (lambda: httpx.Response(500, text="metadata server unavailable"), 503), + (lambda: httpx.Response(403, text="Missing Metadata-Flavor:Google header"), 401), + (lambda: httpx.Response(200, text=""), 401), + (lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503), + (lambda: httpx.ReadTimeout("timed out"), 503), ], - ids=["server error", "empty token", "not on GCP", "timeout"], + ids=["server error", "refused", "empty token", "not on GCP", "timeout"], ) async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, call_path: str, metadata_failure: Callable[[], httpx.Response | httpx.HTTPError], + status_code: int, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(time, "time", lambda: _CLOCK_START) @@ -7379,13 +7385,15 @@ async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_a side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)] ) chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) - with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(litellm.AuthenticationError) as excinfo: + with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises( + (litellm.AuthenticationError, litellm.ServiceUnavailableError) + ) as excinfo: await _ping_vllm_through(call_path, "vllm-sa") sent_while_failing: Final = chat.call_count await _ping_vllm_through(call_path, "vllm-sa") sent_after_recovery: Final = [call.request.headers["Authorization"] for call in chat.calls] - assert excinfo.value.status_code == 401 + assert excinfo.value.status_code == status_code assert "api_key oidc/google secret reference could not be resolved" in excinfo.value.message assert audience.removeprefix("https://") not in excinfo.value.message assert reference in caplog.text @@ -7393,6 +7401,54 @@ async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_a assert sent_after_recovery == [f"Bearer {recovered_token}"] +@pytest.mark.asyncio +async def test_an_oidc_google_api_key_token_cached_when_the_call_starts_is_sent_though_it_expires_mid_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(time, "time", lambda: _CLOCK_START) + audience, reference = _google_reference() + cached_token: Final = _google_id_token(audience, _CLOCK_START) + oidc_cache.set_cache(key=reference, value=cached_token, ttl=60) + clock: Final = itertools.chain([_CLOCK_START + 60], itertools.repeat(_CLOCK_START + 61)) + monkeypatch.setattr(time, "time", lambda: next(clock)) + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue() + + with respx.mock(assert_all_called=False) as upstream: + upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on)) + updates: Final = await acredential_updates({"model": _VLLM_MODEL, "litellm_credential_name": "vllm-sa"}) + + assert updates["api_key"] == cached_token + assert fetched_on.empty() + + +@pytest.mark.asyncio +async def test_a_hung_metadata_server_fails_an_oidc_google_api_key_call_with_a_503_and_leaves_the_default_executor_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + _, reference = _google_reference() + monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)]) + monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1") + loop: Final = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + + with socket.create_server(("127.0.0.1", 0)) as never_answering, respx.mock(assert_all_called=False) as upstream: + monkeypatch.setenv("GCE_METADATA_HOST", f"127.0.0.1:{never_answering.getsockname()[1]}") + upstream.route(host="127.0.0.1").pass_through() + chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) + call: Final = asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa")) + await asyncio.sleep(0.2) + other_work: Final = await asyncio.wait_for(loop.run_in_executor(None, lambda: "done"), timeout=0.5) + with pytest.raises(litellm.ServiceUnavailableError) as excinfo: + await asyncio.wait_for(call, timeout=10) + + assert other_work == "done" + assert excinfo.value.status_code == 503 + assert "the metadata server did not return an ID token" in excinfo.value.message + assert chat.call_count == 0 + + def test_an_oidc_google_api_key_sent_with_the_call_goes_upstream_as_written_without_a_metadata_fetch() -> None: _, reference = _google_reference() From 2d39952b18ad3597bce8d2cf16bf6744b070397e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:03:39 -0700 Subject: [PATCH 4/4] test(oidc): expect 503 when the metadata server answers 5xx in the integration test --- tests/integration/security/test_api_key_secret_reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/security/test_api_key_secret_reference.py b/tests/integration/security/test_api_key_secret_reference.py index 4f69845a046..4fda6365665 100644 --- a/tests/integration/security/test_api_key_secret_reference.py +++ b/tests/integration/security/test_api_key_secret_reference.py @@ -841,7 +841,7 @@ def test_google_reference_fails_closed_when_the_metadata_server_fails(rig: _Rig) marker: Final = _marker() response: Final = _chat(rig.owned.gateway, "wif-google-unavailable", marker) reached: Final = _carrying(rig.provider.requests(), marker) - assert (response.status_code, reached) == (401, ()), (response.text, _bearers(reached)) + assert (response.status_code, reached) == (503, ()), (response.text, _bearers(reached)) assert "oidc/google secret reference could not be resolved" in response.text, response.text assert _UNAVAILABLE_AUDIENCE.removeprefix("https://") not in response.text, response.text assert _UNAVAILABLE_AUDIENCE in _asked_audiences(rig)