This commit is contained in:
Mateo Wang 2026-10-27 16:03:24 -07:00 • committed by GitHub
commit 827962c03e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 544 additions and 44 deletions

View file

@ -35,7 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders
from litellm.utils import resolve_api_key_secret_reference
from litellm.utils import aresolve_api_key_secret_reference
if TYPE_CHECKING:
from prisma import types as prisma_types
@ -227,7 +227,7 @@ async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_clie
return response.status_code == httpx.codes.NOT_FOUND
def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
async def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
model: Final = credentials.get("model")
api_base: Final = credentials.get("api_base")
api_key: Final = credentials.get("api_key")
@ -244,7 +244,7 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str,
return None
if resolved_api_base is None:
return None
return resolved_api_base, resolve_api_key_secret_reference(
return resolved_api_base, await aresolve_api_key_secret_reference(
resolved_api_key, model=model, llm_provider=provider, upstream=credentials
)
@ -255,7 +255,7 @@ async def litellm_executed_provider_for(
provider: Final = litellm_executed_provider_of(credentials)
if provider is None:
return None
upstream: Final = _upstream_of(credentials, provider)
upstream: Final = await _upstream_of(credentials, provider)
if upstream is None:
return None
return provider if await lacks_files_api(*upstream) else None

View file

@ -203,7 +203,7 @@ from litellm.types.utils import (
is_server_derived_pricing_key,
pricing_override_fields,
)
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential
from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential
if TYPE_CHECKING:
from aiohttp import ClientSession
@ -14716,10 +14716,10 @@ def _provider_count_outcome(
return result
def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
async def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params)
try:
load_credentials_from_list(call_params)
return {**call_params, **(await acredential_updates(call_params))}
except litellm.AuthenticationError as e:
raise ProxyException(
message=e.message,
@ -14727,13 +14727,12 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
param="model",
code=401,
) from e
return call_params
def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
async def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
if deployment is None:
return None
return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))}
return {**deployment, "litellm_params": await _with_resolved_api_key(deployment.get("litellm_params", {}))}
_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
@ -14796,7 +14795,7 @@ async def _try_provider_token_count(
model_to_use=model_to_use or "",
messages=messages,
contents=contents,
deployment=_with_resolved_credentials(deployment_in_region),
deployment=await _with_resolved_credentials(deployment_in_region),
request_model=request_model,
tools=tools,
system=system,
@ -14830,7 +14829,7 @@ async def _try_native_responses_count(
model_to_use=model_to_use,
request=responses_request,
policy=policy,
deployment=_with_resolved_credentials(deployment),
deployment=await _with_resolved_credentials(deployment),
request_model=request_model,
)
)

View file

@ -31,7 +31,7 @@ from litellm.types.realtime import (
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
from litellm.utils import ProviderConfigManager, aresolve_api_key_secret_reference
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from ..litellm_core_utils.credential_accessor import CredentialAccessor
@ -810,7 +810,7 @@ async def realtime_health_check(
resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints
str | None, api_base or resolved_params.get("api_base")
)
resolved_api_key: Final = resolve_api_key_secret_reference(
resolved_api_key: Final = await aresolve_api_key_secret_reference(
cast( # cast-ok: provider parameters expose optional string credentials
str | None, api_key or resolved_params.get("api_key")
),

View file

@ -443,6 +443,7 @@ from litellm.utils import (
ProviderConfigManager,
Rules,
TextCompletionResponse,
aresolve_api_key_secret_reference,
function_setup,
get_llm_provider,
get_non_default_completion_params,
@ -453,7 +454,6 @@ from litellm.utils import (
lookup_model_info_for_registration,
model_info_with_pricing_overrides,
provider_rejectable_params,
resolve_api_key_secret_reference,
set_live_deployment_replay,
)
@ -11882,7 +11882,7 @@ class Router:
return
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
return
api_key: Final = resolve_api_key_secret_reference(
api_key: Final = await aresolve_api_key_secret_reference(
params.api_key or dynamic_api_key,
model=model,
llm_provider=provider,

View file

@ -6,7 +6,7 @@ import traceback
from collections.abc import Callable
from dataclasses import dataclass
from functools import lru_cache
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter, ValidationError
@ -167,12 +167,36 @@ def get_secret_str(
LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/")
GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/"
API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX)
_GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal"
_GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS: Final = "3"
_GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token"
def _google_metadata_identity_url() -> str:
host: Final = os.getenv("GCE_METADATA_HOST") or _GOOGLE_METADATA_DEFAULT_HOST
return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity"
def _google_metadata_timeout() -> float:
return float(os.getenv("GCE_METADATA_TIMEOUT") or _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS)
class GoogleMetadataStatusError(ValueError):
def __init__(self, status_code: int) -> None:
super().__init__(f"Google OIDC provider failed with HTTP {status_code}")
self.status_code: Final = status_code
@dataclass(frozen=True, slots=True)
class UnresolvableSecretReference:
kind: str
reason: str
retryable: bool = False
OidcResolver: TypeAlias = Callable[[str], "str | UnresolvableSecretReference"]
_OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"})
@ -198,11 +222,52 @@ def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReferenc
return secret
def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference:
def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretReference:
if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return resolve_local_oidc_reference(value)
try:
token: Final = get_secret_str(value)
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
return _logged_unresolvable(
value,
"oidc/google",
_GOOGLE_TOKEN_UNAVAILABLE,
f"{type(exc).__name__}: {exc}",
retryable=_metadata_server_failed_to_answer(exc),
)
if not token:
return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value")
return token
def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool:
match exc:
case httpx.TimeoutException() | httpx.NetworkError():
return True
case GoogleMetadataStatusError(status_code=status_code):
return status_code >= 500
case _:
return False
def cached_oidc_token(reference: str) -> str | None:
cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped
return cached if isinstance(cached, str) and cached else None
def _logged_unresolvable(
reference: str, kind: str, reason: str, detail: str | None = None, retryable: bool = False
) -> UnresolvableSecretReference:
verbose_logger.warning(
"Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason
"Secret reference %s could not be resolved: %s",
_single_line(reference),
reason if detail is None else f"{reason} ({_single_line(detail)})",
)
return UnresolvableSecretReference(kind=kind, reason=reason)
return UnresolvableSecretReference(kind=kind, reason=reason, retryable=retryable)
def _single_line(text: str) -> str:
return text.replace("\r", "").replace("\n", "")
def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str:
@ -280,24 +345,29 @@ def get_secret(
oidc_client = _get_oidc_http_handler()
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
response = oidc_client.get(
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity",
_google_metadata_identity_url(),
params={"audience": oidc_aud},
headers={"Metadata-Flavor": "Google"},
timeout=_google_metadata_timeout(),
)
if response.status_code == 200:
oidc_token = response.text
ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60)
if ttl > 0:
if not oidc_token:
verbose_logger.warning(
"Google OIDC token for %s is empty; not caching it", _single_line(secret_name)
)
elif ttl > 0:
oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl)
else:
verbose_logger.warning(
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
secret_name,
_single_line(secret_name),
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
)
return oidc_token
else:
raise ValueError("Google OIDC provider failed")
raise GoogleMetadataStatusError(response.status_code)
elif oidc_provider == "circleci":
# https://circleci.com/docs/openid-connect-tokens/
env_secret = os.getenv("CIRCLE_OIDC_TOKEN")

View file

@ -28,9 +28,10 @@ import textwrap
import threading
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext
from dataclasses import dataclass, field
from functools import lru_cache, wraps
from functools import lru_cache, partial, wraps
from importlib import resources
from inspect import iscoroutine
from io import StringIO
@ -476,11 +477,14 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig
from litellm.secret_managers.main import (
LOCAL_OIDC_SECRET_PREFIXES,
API_KEY_OIDC_SECRET_PREFIXES,
GOOGLE_OIDC_SECRET_PREFIX,
OidcResolver,
UnresolvableSecretReference,
cached_oidc_token,
get_secret,
oidc_reference_kind,
resolve_local_oidc_reference,
resolve_api_key_oidc_reference,
)
from ._logging import is_debugging_on, verbose_logger
@ -777,19 +781,56 @@ def resolve_api_key_secret_reference(
return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream)
async def aresolve_api_key_secret_reference(
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
) -> str | None:
if not isinstance(api_key, AdminApiKeySecretReference):
return api_key
return await _resolved_off_the_event_loop(
partial(_secret_behind_api_key_reference, api_key, model, llm_provider, upstream), api_key
)
_OIDC_TOKEN_FETCH_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-oidc-token-fetch")
_Resolved: Final = TypeVar("_Resolved")
async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolved], api_key: object) -> _Resolved:
if not isinstance(api_key, AdminApiKeySecretReference) or not api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return resolve(resolve_api_key_oidc_reference)
match cached_oidc_token(api_key):
case str() as token:
return resolve(lambda _reference: token)
case None:
caller_context: Final = contextvars.copy_context()
return await asyncio.get_running_loop().run_in_executor(
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference)
)
def _secret_behind_api_key_reference(
reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object]
reference: AdminApiKeySecretReference,
model: object,
llm_provider: object,
upstream: Mapping[str, object],
resolve_oidc: OidcResolver = resolve_api_key_oidc_reference,
) -> str:
if not reference.may_reach(upstream):
raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model)
if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES):
if not reference.startswith(API_KEY_OIDC_SECRET_PREFIXES):
raise _secret_reference_error(
f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. "
"Use oidc/file/<absolute path> or oidc/env/<VARIABLE_NAME>.",
"Use oidc/file/<absolute path>, oidc/env/<VARIABLE_NAME> or oidc/google/<audience>.",
llm_provider=llm_provider,
model=model,
)
match resolve_local_oidc_reference(reference):
match resolve_oidc(reference):
case UnresolvableSecretReference(kind=kind, reason=reason, retryable=True):
raise ServiceUnavailableError(
message=f"api_key {kind} secret reference could not be resolved: {reason}",
llm_provider=llm_provider,
model=model,
)
case UnresolvableSecretReference(kind=kind, reason=reason):
raise _secret_reference_error(
f"api_key {kind} secret reference could not be resolved: {reason}",
@ -826,16 +867,31 @@ def load_credentials_from_list(kwargs: dict[str, object]) -> None:
Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key
reference for the secret it points to, read now so a rotated token is used on the next call
"""
kwargs.update(_credential_updates(kwargs))
kwargs.update(_credential_updates(kwargs, _named_credential_values(kwargs)))
def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]:
async def acredential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]:
credential_values: Final = _named_credential_values(kwargs)
api_key: Final = kwargs.get("api_key", credential_values.get("api_key"))
return await _resolved_off_the_event_loop(
partial(_credential_updates, kwargs, credential_values), _api_key_after_credentials(kwargs, credential_values)
)
def _api_key_after_credentials(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> object:
return kwargs.get("api_key", credential_values.get("api_key"))
def _credential_updates(
kwargs: Mapping[str, object],
credential_values: Mapping[str, object],
resolve_oidc: OidcResolver = resolve_api_key_oidc_reference,
) -> Mapping[str, object]:
api_key: Final = _api_key_after_credentials(kwargs, credential_values)
if not isinstance(api_key, AdminApiKeySecretReference):
return credential_values
secret: Final = _secret_behind_api_key_reference(
api_key,
resolve_oidc=resolve_oidc,
model=kwargs.get("model"),
llm_provider=kwargs.get("custom_llm_provider"),
upstream=MappingProxyType(
@ -2216,7 +2272,7 @@ def client(original_function):
kwargs["litellm_logging_obj"] = logging_obj
## LOAD CREDENTIALS
load_credentials_from_list(kwargs)
kwargs.update(await acredential_updates(kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is the untyped request dict
from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session
await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict

View file

@ -1,6 +1,8 @@
from __future__ import annotations
import base64
import json
import time
import uuid
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
@ -46,7 +48,17 @@ _SEARCH_PATH: Final = f"/v1/vector_stores/{_VECTOR_STORE}/search"
_BATCH_DEPLOYMENT: Final = "wif-vllm-batch"
_BEDROCK_MODEL: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5"
_PROVIDER_INPUT_TOKENS: Final = 7919
_UNSUPPORTED_REFERENCE: Final = "oidc/google/https://example.invalid"
_UNSUPPORTED_REFERENCE: Final = "oidc/github/https://example.invalid"
_GOOGLE_MODEL: Final = "hosted_vllm/wif-google"
_GOOGLE_AUDIENCES: Final = MappingProxyType(
{
"config_inline": "https://vllm-inline.example.test",
"config_credential": "https://vllm-credential.example.test",
"db_credential": "https://vllm-db-credential.example.test",
}
)
_UNAVAILABLE_AUDIENCE: Final = "https://vllm-unavailable.example.test"
_IDENTITY_PATH: Final = "/computeMetadata/v1/instance/service-accounts/default/identity"
_RUN_SECRETS_TOKEN: Final = "/run/secrets/wif-absent-token"
_SOURCES: Final = ("config_credential", "config_inline", "db_credential", "db_inline")
_UNREADABLE: Final = MappingProxyType(
@ -108,6 +120,43 @@ def _reference(path: Path) -> str:
return f"oidc/file/{path}"
def _google_reference(audience: str) -> str:
return f"oidc/google/{audience}"
def _jwt_segment(part: Mapping[str, JsonValue]) -> str:
return base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=")
@dataclass(frozen=True, slots=True)
class _Google:
expires: int
def token(self, audience: str) -> str:
claims: Final = _jwt_segment({"iss": "https://accounts.google.com", "aud": audience, "exp": self.expires})
return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{claims}.signature"
def bearers(self) -> frozenset[str]:
return frozenset(f"Bearer {self.token(audience)}" for audience in _GOOGLE_AUDIENCES.values())
def _metadata_server(google: _Google) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
target: Final = httpx.URL(request.target)
audience: Final = target.params.get("audience")
if audience is None or (request.method, target.path, request.headers.get("metadata-flavor")) != (
"GET",
_IDENTITY_PATH,
"Google",
):
return Reply(status=404, body=b"not found", content_type="text/plain")
if audience == _UNAVAILABLE_AUDIENCE:
return Reply(status=503, body=b"metadata server unavailable", content_type="text/plain")
return Reply(body=google.token(audience).encode(), content_type="text/plain")
return respond
def _secrets(directory: Path) -> _Secrets:
allowed: Final = directory / "secrets"
outside_dir: Final = directory / "outside"
@ -169,9 +218,9 @@ def _served(request: Request) -> Reply:
return openai_error(404)
def _provider(secrets: _Secrets) -> Callable[[Request], Reply]:
def _provider(secrets: _Secrets, google: _Google) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
if request.headers.get("authorization") != f"Bearer {secrets.current()}":
if request.headers.get("authorization") not in {f"Bearer {secrets.current()}", *google.bearers()}:
return openai_error(401)
return _served(request)
@ -224,6 +273,19 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path:
"aws_bedrock_runtime_endpoint": provider.removesuffix("/v1"),
},
),
_deployment(
"wif-google-inline",
{
"model": _GOOGLE_MODEL,
"api_key": _google_reference(_GOOGLE_AUDIENCES["config_inline"]),
"api_base": provider,
},
),
_deployment("wif-google-credential", _named(_GOOGLE_MODEL, "wif-google")),
_deployment(
"wif-google-unavailable",
{"model": _GOOGLE_MODEL, "api_key": _google_reference(_UNAVAILABLE_AUDIENCE), "api_base": provider},
),
_deployment(
"wif-mistral-passthrough",
{
@ -239,6 +301,7 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path:
_credential("wif-outside", _reference(secrets.outside), provider),
_credential("wif-unsupported", _UNSUPPORTED_REFERENCE, provider),
_credential("wif-run-secrets", f"oidc/file/{_RUN_SECRETS_TOKEN}", provider),
_credential("wif-google", _google_reference(_GOOGLE_AUDIENCES["config_credential"]), provider),
],
"vector_store_registry": [
{
@ -312,6 +375,8 @@ class _Rig:
owned: OwnedProxy
provider: Recorder
intruder: Recorder
metadata: Recorder
google: _Google
provider_base: str
intruder_base: str
secrets: _Secrets
@ -339,17 +404,23 @@ def _is_discovery(request: Request) -> bool:
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("api_key_secret_reference").resolve()
secrets: Final = _secrets(directory)
google: Final = _Google(expires=int(time.time()) + 3600)
with (
gateway_from_environment() as gateway,
wire_server(_provider(secrets)) as provider_wire,
wire_server(_provider(secrets, google)) as provider_wire,
wire_server(_intruder) as intruder_wire,
wire_server(_metadata_server(google)) as metadata_wire,
):
provider: Final = Recorder(provider_wire)
provider_base: Final = f"{provider_wire.url}/v1"
with owned_proxy_process(
gateway,
directory,
{**secrets.overrides(), "MISTRAL_API_BASE": provider_wire.url},
{
**secrets.overrides(),
"MISTRAL_API_BASE": provider_wire.url,
"GCE_METADATA_HOST": httpx.URL(metadata_wire.url).netloc.decode(),
},
config=_config(directory, secrets, provider_base),
workers=2,
healthcheck=WorkerHealthcheck.LENIENT,
@ -361,6 +432,8 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
owned=owned,
provider=provider,
intruder=Recorder(intruder_wire),
metadata=Recorder(metadata_wire),
google=google,
provider_base=provider_base,
intruder_base=f"{intruder_wire.url}/v1",
secrets=secrets,
@ -708,7 +781,70 @@ def test_team_row_naming_the_credential_sends_the_token_only_to_its_host(rig: _R
def test_startup_discovery_sends_the_boot_token(rig: _Rig) -> None:
sent: Final = frozenset(_bearers(tuple(request for request in rig.boot if _is_discovery(request))))
assert sent == {f"Bearer {rig.boot_token}"}, sent
assert sent - rig.google.bearers() == {f"Bearer {rig.boot_token}"}, sent
def _discovery_bearers(rig: _Rig) -> frozenset[str | None]:
return frozenset(_bearers(tuple(request for request in rig.provider.requests() if _is_discovery(request))))
def test_startup_discovery_sends_the_google_token(rig: _Rig) -> None:
expected: Final = frozenset(
f"Bearer {rig.google.token(_GOOGLE_AUDIENCES[source])}" for source in ("config_inline", "config_credential")
)
sent: Final = eventually(
partial(_discovery_bearers, rig), lambda seen: expected <= seen, seconds=30, return_last_on_timeout=True
)
assert expected <= sent, sent
def _google_model(rig: _Rig, scenario: Scenario, source: str) -> str:
match source:
case "config_inline":
return "wif-google-inline"
case "config_credential":
return "wif-google-credential"
case "db_credential":
credential: Final = f"db-google-{uuid.uuid4().hex}"
rig.owned.gateway.post(
"/credentials",
_credential(credential, _google_reference(_GOOGLE_AUDIENCES[source]), rig.provider_base),
)
scenario.cleanups.callback(_delete_credential, rig.owned.gateway, credential)
name, identity = _created_model(
rig.owned.gateway,
scenario,
_deployment(f"db-google-{uuid.uuid4().hex}", _named(_GOOGLE_MODEL, credential)),
)
eventually(partial(rig.owned.serves_only_on_every_worker, name, identity), bool, seconds=120)
return name
case _:
pytest.fail(f"unknown source {source}")
def _asked_audiences(rig: _Rig) -> frozenset[str | None]:
return frozenset(httpx.URL(request.target).params.get("audience") for request in rig.metadata.requests())
@pytest.mark.parametrize("source", tuple(_GOOGLE_AUDIENCES))
def test_google_reference_sends_the_metadata_server_token_for_its_audience(rig: _Rig, source: str) -> None:
audience: Final = _GOOGLE_AUDIENCES[source]
with rig.owned.gateway.scenario() as scenario:
marker: Final = _marker()
response: Final = _chat(rig.owned.gateway, _google_model(rig, scenario, source), marker)
sent: Final = _bearers(_carrying(rig.provider.requests(), marker))
assert (response.status_code, sent) == (200, (f"Bearer {rig.google.token(audience)}",)), response.text
assert audience in _asked_audiences(rig)
def test_google_reference_fails_closed_when_the_metadata_server_fails(rig: _Rig) -> None:
marker: Final = _marker()
response: Final = _chat(rig.owned.gateway, "wif-google-unavailable", marker)
reached: Final = _carrying(rig.provider.requests(), marker)
assert (response.status_code, reached) == (503, ()), (response.text, _bearers(reached))
assert "oidc/google secret reference could not be resolved" in response.text, response.text
assert _UNAVAILABLE_AUDIENCE.removeprefix("https://") not in response.text, response.text
assert _UNAVAILABLE_AUDIENCE in _asked_audiences(rig)
def _side_call(rig: _Rig, seam: str, marker: str) -> tuple[httpx.Response, Callable[[Request], bool]]:

View file

@ -3,10 +3,12 @@ import json
import logging
import os
import time
import uuid
from typing import Final
from unittest.mock import Mock, patch
import pytest
import respx
import litellm
from litellm._logging import redact_internal_details_from_client_message
@ -32,7 +34,7 @@ class MockHTTPHandler:
self.text = "mocked_token"
self.json_data = {"value": "mocked_token"}
def get(self, url, params=None, headers=None):
def get(self, url, params=None, headers=None, timeout=None):
# Store params for audience verification
self.last_params = params
logger.debug(
@ -224,6 +226,23 @@ def test_oidc_google_failure():
get_secret(secret_name)
def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_names(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("GCE_METADATA_HOST", "127.0.0.1:8930")
audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}"
with respx.mock(assert_all_called=True) as metadata:
metadata.get(
"http://127.0.0.1:8930/computeMetadata/v1/instance/service-accounts/default/identity",
params={"audience": audience},
headers={"Metadata-Flavor": "Google"},
).respond(200, text="metadata-host-token")
token: Final = get_secret(f"oidc/google/{audience}")
assert token == "metadata-host-token"
def test_oidc_circleci_success(monkeypatch):
monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token")

View file

@ -5,14 +5,19 @@ import copy
import contextlib
import contextvars
import io
import itertools
import json
import logging
import os
import queue
import socket
import threading
import time
import uuid
from collections.abc import Callable, Iterator, Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from functools import partial
from pathlib import Path, PurePath
from typing import Final, SupportsIndex, cast
from unittest import mock
@ -79,8 +84,10 @@ from litellm.types.utils import (
all_litellm_params,
bedrock_batch_litellm_params,
)
from litellm.secret_managers.main import oidc_cache
from litellm.types.videos.main import VideoObject
from litellm.utils import (
acredential_updates,
_invalidate_model_cost_lowercase_map,
add_custom_logger_callback_to_specific_event,
check_valid_key,
@ -7318,7 +7325,6 @@ def test_unresolvable_oidc_api_key_fails_closed_naming_the_reason_without_its_ta
("reference", "kind"),
[
("oidc/env_path/MISTRAL_SA_TOKEN_PATH", "oidc/env_path"),
("oidc/google/https://api.mistral.ai", "oidc/google"),
("oidc/github/mistral", "oidc/github"),
("oidc/env_MISTRAL_SA_TOKEN_PATH", "oidc/<unknown kind>"),
("oidc/LIT-UNKNOWN-KIND/MISTRAL_SA_TOKEN_PATH", "oidc/<unknown kind>"),
@ -7339,6 +7345,7 @@ def test_oidc_api_key_of_a_kind_api_key_cannot_resolve_fails_closed_as_an_authen
assert not route.called
assert excinfo.value.status_code == 401
assert f"{kind} secret reference" in excinfo.value.message
assert "oidc/google/<audience>" in excinfo.value.message
assert reference.removeprefix(kind).lstrip("/") not in excinfo.value.message
assert "MISTRAL_SA_TOKEN_PATH" not in excinfo.value.message
@ -7354,7 +7361,7 @@ async def test_router_fails_over_from_a_deployment_whose_oidc_api_key_kind_canno
"model_name": "devstral",
"litellm_params": {
"model": "mistral/devstral-latest",
"api_key": "oidc/google/https://api.mistral.ai",
"api_key": "oidc/github/mistral",
"order": 1,
},
},
@ -7393,6 +7400,219 @@ def test_an_oidc_api_key_sent_with_the_call_goes_upstream_as_written(
assert sent == f"Bearer {reference}"
_GOOGLE_METADATA_IDENTITY: Final = (
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity"
)
_VLLM_BASE: Final = "https://vllm.example.test/v1"
_VLLM_CHAT_URL: Final = f"{_VLLM_BASE}/chat/completions"
_VLLM_MODEL: Final = "hosted_vllm/qwen3-0.6b"
_VLLM_REPLY: Final = {**_MISTRAL_REPLY, "model": "qwen3-0.6b"}
_CLOCK_START: Final = 1_900_000_000.0
_GOOGLE_TOKEN_LIFETIME: Final = 3600
def _jwt_segment(claims: Mapping[str, object]) -> str:
return base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=")
def _google_id_token(audience: str, issued_at: float) -> str:
claims: Final = {"aud": audience, "iat": int(issued_at), "exp": int(issued_at) + _GOOGLE_TOKEN_LIFETIME}
return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{_jwt_segment(claims)}.signature"
def _google_reference() -> tuple[str, str]:
audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}"
return audience, f"oidc/google/{audience}"
def _metadata_identity(fetched_on: queue.SimpleQueue[int], request: httpx.Request) -> httpx.Response:
fetched_on.put(threading.get_ident())
if request.headers.get("Metadata-Flavor") != "Google":
return httpx.Response(403)
return httpx.Response(200, text=_google_id_token(request.url.params["audience"], time.time()))
def _vllm_credential(api_key: str) -> CredentialItem:
return CredentialItem(
credential_name="vllm-sa", credential_values={"api_key": api_key, "api_base": _VLLM_BASE}, credential_info={}
)
async def _ping_vllm_through(call_path: str, credential_name: str) -> None:
if call_path == "async":
await litellm.acompletion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name)
return
litellm.completion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name)
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["sync", "async"])
async def test_named_credential_oidc_google_api_key_sends_the_metadata_token_and_refreshes_it_before_it_expires(
monkeypatch: pytest.MonkeyPatch, call_path: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
audience, reference = _google_reference()
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
refreshed_at: Final = _CLOCK_START + _GOOGLE_TOKEN_LIFETIME - 59
with respx.mock(assert_all_called=True) as upstream:
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock(
side_effect=partial(_metadata_identity, queue.SimpleQueue())
)
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
await _ping_vllm_through(call_path, "vllm-sa")
await _ping_vllm_through(call_path, "vllm-sa")
fetches_within_lifetime: Final = metadata.call_count
monkeypatch.setattr(time, "time", lambda: refreshed_at)
await _ping_vllm_through(call_path, "vllm-sa")
sent: Final = [call.request.headers["Authorization"] for call in chat.calls]
audiences: Final = [call.request.url.params["audience"] for call in metadata.calls]
first_token: Final = f"Bearer {_google_id_token(audience, _CLOCK_START)}"
assert sent == [first_token, first_token, f"Bearer {_google_id_token(audience, refreshed_at)}"]
assert (fetches_within_lifetime, audiences) == (1, [audience, audience])
assert litellm.credential_list[0].credential_values["api_key"] == reference
@pytest.mark.asyncio
async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_and_keeps_the_reference(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
audience, reference = _google_reference()
fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
router: Final = litellm.Router(
model_list=[
{
"model_name": "vllm",
"litellm_params": {"model": _VLLM_MODEL, "api_base": _VLLM_BASE, "api_key": reference},
}
]
)
try:
with respx.mock(assert_all_called=True) as upstream:
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on))
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
await router.acompletion(model="vllm", messages=_PING)
sent: Final = chat.calls.last.request.headers["Authorization"]
finally:
router.discard()
assert sent == f"Bearer {_google_id_token(audience, _CLOCK_START)}"
assert fetched_on.qsize() == 1
assert fetched_on.get_nowait() != threading.get_ident()
assert router.get_model_list(model_name="vllm")[0]["litellm_params"]["api_key"] == reference
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["sync", "async"])
@pytest.mark.parametrize(
("metadata_failure", "status_code"),
[
(lambda: httpx.Response(500, text="metadata server unavailable"), 503),
(lambda: httpx.Response(403, text="Missing Metadata-Flavor:Google header"), 401),
(lambda: httpx.Response(200, text=""), 401),
(lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503),
(lambda: httpx.ReadTimeout("timed out"), 503),
],
ids=["server error", "refused", "empty token", "not on GCP", "timeout"],
)
async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
call_path: str,
metadata_failure: Callable[[], httpx.Response | httpx.HTTPError],
status_code: int,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
audience, reference = _google_reference()
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
recovered_token: Final = _google_id_token(audience, _CLOCK_START)
with respx.mock(assert_all_called=True) as upstream:
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(
side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)]
)
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(
(litellm.AuthenticationError, litellm.ServiceUnavailableError)
) as excinfo:
await _ping_vllm_through(call_path, "vllm-sa")
sent_while_failing: Final = chat.call_count
await _ping_vllm_through(call_path, "vllm-sa")
sent_after_recovery: Final = [call.request.headers["Authorization"] for call in chat.calls]
assert excinfo.value.status_code == status_code
assert "api_key oidc/google secret reference could not be resolved" in excinfo.value.message
assert audience.removeprefix("https://") not in excinfo.value.message
assert reference in caplog.text
assert sent_while_failing == 0
assert sent_after_recovery == [f"Bearer {recovered_token}"]
@pytest.mark.asyncio
async def test_an_oidc_google_api_key_token_cached_when_the_call_starts_is_sent_though_it_expires_mid_call(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
audience, reference = _google_reference()
cached_token: Final = _google_id_token(audience, _CLOCK_START)
oidc_cache.set_cache(key=reference, value=cached_token, ttl=60)
clock: Final = itertools.chain([_CLOCK_START + 60], itertools.repeat(_CLOCK_START + 61))
monkeypatch.setattr(time, "time", lambda: next(clock))
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
with respx.mock(assert_all_called=False) as upstream:
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on))
updates: Final = await acredential_updates({"model": _VLLM_MODEL, "litellm_credential_name": "vllm-sa"})
assert updates["api_key"] == cached_token
assert fetched_on.empty()
@pytest.mark.asyncio
async def test_a_hung_metadata_server_fails_an_oidc_google_api_key_call_with_a_503_and_leaves_the_default_executor_free(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
_, reference = _google_reference()
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1")
loop: Final = asyncio.get_running_loop()
loop.set_default_executor(ThreadPoolExecutor(max_workers=1))
with socket.create_server(("127.0.0.1", 0)) as never_answering, respx.mock(assert_all_called=False) as upstream:
monkeypatch.setenv("GCE_METADATA_HOST", f"127.0.0.1:{never_answering.getsockname()[1]}")
upstream.route(host="127.0.0.1").pass_through()
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
call: Final = asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa"))
await asyncio.sleep(0.2)
other_work: Final = await asyncio.wait_for(loop.run_in_executor(None, lambda: "done"), timeout=0.5)
with pytest.raises(litellm.ServiceUnavailableError) as excinfo:
await asyncio.wait_for(call, timeout=10)
assert other_work == "done"
assert excinfo.value.status_code == 503
assert "the metadata server did not return an ID token" in excinfo.value.message
assert chat.call_count == 0
def test_an_oidc_google_api_key_sent_with_the_call_goes_upstream_as_written_without_a_metadata_fetch() -> None:
_, reference = _google_reference()
with respx.mock(assert_all_called=True) as upstream:
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
litellm.completion(model=_VLLM_MODEL, api_base=_VLLM_BASE, messages=_PING, api_key=reference)
sent: Final = chat.calls.last.request.headers["Authorization"]
assert sent == f"Bearer {reference}"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model_info", "expected"),