fix(secret_managers): keep get_secret's metadata wait, share concurrent google token fetches, refuse oidc/google on pass-through

A model's api_key fetch keeps the 3 second GCE_METADATA_TIMEOUT wait while get_secret callers keep the handler default. Any metadata transport error answers a retryable 503. Async cache misses for one reference share one fetch, and a call the reference may not go to is refused before any fetch. Pass-through routes answer 401 for an oidc/google credential instead of sending the reference text. The metadata client is closed after each fetch, and the unused sync resolver is removed.
This commit is contained in:
mateo-berri 2026-10-28 19:24:14 -07:00
parent 2087d8bd89
commit efe52b79b5
6 changed files with 183 additions and 47 deletions

View file

@ -11,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
get_secret_str,
resolve_local_oidc_reference,
@ -46,6 +47,14 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str |
def _resolve_oidc_reference(credential: str | None) -> str | None:
if credential is None:
return None
if credential.startswith(GOOGLE_OIDC_SECRET_PREFIX):
raise HTTPException(
status_code=401,
detail=(
"pass-through credential oidc/google secret reference could not be resolved: pass-through routes "
"do not fetch Google ID tokens. Use a static key, oidc/file/<absolute path> or oidc/env/<VARIABLE_NAME>."
),
)
match resolve_local_oidc_reference(credential):
case UnresolvableSecretReference(kind=kind, reason=reason):
raise HTTPException(

View file

@ -228,7 +228,9 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere
if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return resolve_local_oidc_reference(value)
try:
token: Final = get_secret_str(value)
token: Final = _google_id_token(
value, value.removeprefix(GOOGLE_OIDC_SECRET_PREFIX), timeout=_google_metadata_timeout()
)
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
return _logged_unresolvable(
value,
@ -244,7 +246,7 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere
def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool:
match exc:
case httpx.TimeoutException() | httpx.NetworkError():
case httpx.TransportError():
return True
case GoogleMetadataStatusError(status_code=status_code):
return status_code >= 500
@ -252,6 +254,38 @@ def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL |
return False
def _google_id_token(reference: str, audience: str, timeout: float | None) -> str:
cached: Final = cached_oidc_token(reference)
if cached is not None:
return cached
oidc_client: Final = _get_oidc_http_handler()
try:
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
response: Final = oidc_client.get(
_google_metadata_identity_url(),
params={"audience": audience},
headers={"Metadata-Flavor": "Google"},
timeout=timeout,
)
finally:
oidc_client.close()
if response.status_code != 200:
raise GoogleMetadataStatusError(response.status_code)
token: Final = response.text
ttl: Final = _oidc_token_cache_ttl(token, 3600 - 60)
if not token:
verbose_logger.warning("Google OIDC token for %s is empty; not caching it", _single_line(reference))
elif ttl > 0:
oidc_cache.set_cache(key=reference, value=token, ttl=ttl)
else:
verbose_logger.warning(
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
_single_line(reference),
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
)
return token
def cached_oidc_token(reference: str) -> str | None:
cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped
return cached if isinstance(cached, str) and cached else None
@ -340,36 +374,7 @@ def get_secret(
oidc_aud = "/".join(secret_name_split.split("/")[1:])
# TODO: Add caching for HTTP requests
if oidc_provider == "google":
oidc_token = oidc_cache.get_cache(key=secret_name)
if oidc_token is not None:
return oidc_token
oidc_client = _get_oidc_http_handler()
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
response = oidc_client.get(
_google_metadata_identity_url(),
params={"audience": oidc_aud},
headers={"Metadata-Flavor": "Google"},
timeout=_google_metadata_timeout(),
)
if response.status_code == 200:
oidc_token = response.text
ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60)
if not oidc_token:
verbose_logger.warning(
"Google OIDC token for %s is empty; not caching it", _single_line(secret_name)
)
elif ttl > 0:
oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl)
else:
verbose_logger.warning(
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
_single_line(secret_name),
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
)
return oidc_token
else:
raise GoogleMetadataStatusError(response.status_code)
return _google_id_token(secret_name, oidc_aud, timeout=None)
elif oidc_provider == "circleci":
# https://circleci.com/docs/openid-connect-tokens/
env_secret = os.getenv("CIRCLE_OIDC_TOKEN")

View file

@ -773,14 +773,6 @@ def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, objec
)
def resolve_api_key_secret_reference(
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
) -> str | None:
if not isinstance(api_key, AdminApiKeySecretReference):
return api_key
return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream)
async def aresolve_api_key_secret_reference(
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
) -> str | None:
@ -802,10 +794,40 @@ async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolv
case str() as token:
return resolve(lambda _reference: token)
case None:
caller_context: Final = contextvars.copy_context()
return await asyncio.get_running_loop().run_in_executor(
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference)
_raise_if_the_reference_may_not_be_used(resolve)
fetched: Final = await asyncio.shield(
_google_token_fetch(asyncio.get_running_loop(), api_key).start_or_join()
)
return resolve(lambda _reference: fetched)
def _raise_if_the_reference_may_not_be_used(resolve: Callable[[OidcResolver], object]) -> None:
_ = resolve(lambda _reference: "")
@dataclass(slots=True)
class _GoogleTokenFetch:
reference: str
in_flight: asyncio.Future[str | UnresolvableSecretReference] | None = None
def start_or_join(self) -> asyncio.Future[str | UnresolvableSecretReference]:
if self.in_flight is not None:
return self.in_flight
caller_context: Final = contextvars.copy_context()
fetch: Final = asyncio.get_running_loop().run_in_executor(
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve_api_key_oidc_reference, self.reference)
)
fetch.add_done_callback(self._land)
self.in_flight = fetch
return fetch
def _land(self, _fetch: asyncio.Future[str | UnresolvableSecretReference]) -> None:
self.in_flight = None
@lru_cache(maxsize=256)
def _google_token_fetch(loop: asyncio.AbstractEventLoop, reference: str) -> _GoogleTokenFetch:
return _GoogleTokenFetch(reference)
def _secret_behind_api_key_reference(

View file

@ -485,7 +485,7 @@ def test_oidc_env_path_reference_is_not_resolved(tmp_path, monkeypatch):
)
def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch):
def test_oidc_google_reference_is_refused_without_a_metadata_fetch(monkeypatch):
def fail(*args, **kwargs):
raise AssertionError("network-backed oidc reference was resolved inside the request path")
@ -494,11 +494,24 @@ def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch):
lambda name, *a, **k: "oidc/google/https://example.com" if name == "OPENAI_API_KEY" else fail(),
)
assert (
with pytest.raises(HTTPException) as excinfo:
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
== "oidc/google/https://example.com"
assert excinfo.value.status_code == 401
assert "pass-through credential oidc/google" in str(excinfo.value.detail)
assert "example.com" not in str(excinfo.value.detail)
def test_deployment_oidc_google_reference_is_not_sent_upstream(monkeypatch):
llm_router = litellm.Router(
model_list=[_flagged_deployment("openai/gpt-4o", api_key="oidc/google/https://vllm.example.test")]
)
with pytest.raises(HTTPException) as excinfo:
_passthrough_router(llm_router).get_credentials(custom_llm_provider="openai", region_name=None)
assert excinfo.value.status_code == 401
@pytest.fixture()
async def _drain_logging_worker():

View file

@ -51,6 +51,9 @@ class MockHTTPHandler:
mock_response.json.return_value = self.json_data
return mock_response
def close(self):
pass
@pytest.fixture
def mock_oidc_cache():
@ -248,6 +251,24 @@ def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_
assert token == "metadata-host-token"
def test_oidc_google_secret_outside_a_model_api_key_keeps_its_metadata_wait_when_gce_metadata_timeout_is_set(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1")
audience: Final = f"https://sts.example.test/{uuid.uuid4().hex}"
with respx.mock(assert_all_called=True) as metadata:
identity: Final = metadata.get(
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity",
params={"audience": audience},
).respond(200, text="web-identity-token")
token: Final = get_secret(f"oidc/google/{audience}")
waits: Final = identity.calls.last.request.extensions["timeout"]
assert token == "web-identity-token"
assert waits == {"connect": 5.0, "read": 600.0, "write": 600.0, "pool": 600.0}
def test_oidc_circleci_success(monkeypatch):
monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token")

View file

@ -7518,8 +7518,9 @@ async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_an
(lambda: httpx.Response(200, text=""), 401),
(lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503),
(lambda: httpx.ReadTimeout("timed out"), 503),
(lambda: httpx.RemoteProtocolError("Server disconnected without sending a response."), 503),
],
ids=["server error", "refused", "empty token", "not on GCP", "timeout"],
ids=["server error", "refused", "empty token", "not on GCP", "timeout", "disconnected"],
)
async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call(
monkeypatch: pytest.MonkeyPatch,
@ -7610,6 +7611,71 @@ async def test_a_pending_oidc_google_metadata_fetch_leaves_the_default_executor_
assert chat.call_count == 0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"metadata_answer",
[lambda token: httpx.Response(200, text=token), lambda _token: httpx.Response(500, text="metadata server down")],
ids=["token", "outage"],
)
async def test_concurrent_calls_that_miss_the_oidc_google_token_cache_share_one_metadata_fetch(
monkeypatch: pytest.MonkeyPatch, metadata_answer: Callable[[str], httpx.Response]
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
audience, reference = _google_reference()
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
token: Final = _google_id_token(audience, _CLOCK_START)
fetch_started: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
release: Final = threading.Event()
def held_metadata(_request: httpx.Request) -> httpx.Response:
fetch_started.put(threading.get_ident())
release.wait(timeout=30)
return metadata_answer(token)
with ThreadPoolExecutor(max_workers=1) as observer, respx.mock(assert_all_called=False) as upstream:
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=held_metadata)
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
calls: Final = [asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa")) for _ in range(24)]
try:
await asyncio.wait_for(asyncio.get_running_loop().run_in_executor(observer, fetch_started.get), timeout=30)
await asyncio.sleep(0.5)
finally:
release.set()
outcomes: Final = await asyncio.gather(*calls, return_exceptions=True)
sent: Final = [call.request.headers["Authorization"] for call in chat.calls]
assert metadata.call_count == 1
if metadata.calls.last.response.status_code == 200:
assert outcomes == [None] * 24
assert sent == [f"Bearer {token}"] * 24
else:
assert [getattr(outcome, "status_code", None) for outcome in outcomes] == [503] * 24
assert sent == []
@pytest.mark.asyncio
async def test_a_call_an_oidc_google_api_key_may_not_go_to_is_refused_without_a_metadata_fetch(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_, reference = _google_reference()
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
with respx.mock(assert_all_called=False) as upstream:
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).respond(500)
with pytest.raises(litellm.AuthenticationError) as excinfo:
await acredential_updates(
{
"model": _VLLM_MODEL,
"litellm_credential_name": "vllm-sa",
"api_base": "https://elsewhere.example.test/v1",
}
)
assert "only to that credential's own api_base" in excinfo.value.message
assert metadata.call_count == 0
@pytest.mark.parametrize(
("configured", "bound"),
[(None, 3.0), ("1.5", 1.5), ("three seconds", 3.0), ("0", 3.0), ("-2", 3.0)],