mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 2d39952b18 into 969161501a
This commit is contained in:
commit
827962c03e
9 changed files with 544 additions and 44 deletions
|
|
@ -35,7 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository
|
|||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose
|
||||
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders
|
||||
from litellm.utils import resolve_api_key_secret_reference
|
||||
from litellm.utils import aresolve_api_key_secret_reference
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import types as prisma_types
|
||||
|
|
@ -227,7 +227,7 @@ async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_clie
|
|||
return response.status_code == httpx.codes.NOT_FOUND
|
||||
|
||||
|
||||
def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
|
||||
async def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
|
||||
model: Final = credentials.get("model")
|
||||
api_base: Final = credentials.get("api_base")
|
||||
api_key: Final = credentials.get("api_key")
|
||||
|
|
@ -244,7 +244,7 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str,
|
|||
return None
|
||||
if resolved_api_base is None:
|
||||
return None
|
||||
return resolved_api_base, resolve_api_key_secret_reference(
|
||||
return resolved_api_base, await aresolve_api_key_secret_reference(
|
||||
resolved_api_key, model=model, llm_provider=provider, upstream=credentials
|
||||
)
|
||||
|
||||
|
|
@ -255,7 +255,7 @@ async def litellm_executed_provider_for(
|
|||
provider: Final = litellm_executed_provider_of(credentials)
|
||||
if provider is None:
|
||||
return None
|
||||
upstream: Final = _upstream_of(credentials, provider)
|
||||
upstream: Final = await _upstream_of(credentials, provider)
|
||||
if upstream is None:
|
||||
return None
|
||||
return provider if await lacks_files_api(*upstream) else None
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ from litellm.types.utils import (
|
|||
is_server_derived_pricing_key,
|
||||
pricing_override_fields,
|
||||
)
|
||||
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential
|
||||
from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
|
@ -14716,10 +14716,10 @@ def _provider_count_outcome(
|
|||
return result
|
||||
|
||||
|
||||
def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
|
||||
async def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
|
||||
call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params)
|
||||
try:
|
||||
load_credentials_from_list(call_params)
|
||||
return {**call_params, **(await acredential_updates(call_params))}
|
||||
except litellm.AuthenticationError as e:
|
||||
raise ProxyException(
|
||||
message=e.message,
|
||||
|
|
@ -14727,13 +14727,12 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
|
|||
param="model",
|
||||
code=401,
|
||||
) from e
|
||||
return call_params
|
||||
|
||||
|
||||
def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
|
||||
async def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
|
||||
if deployment is None:
|
||||
return None
|
||||
return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))}
|
||||
return {**deployment, "litellm_params": await _with_resolved_api_key(deployment.get("litellm_params", {}))}
|
||||
|
||||
|
||||
_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -14796,7 +14795,7 @@ async def _try_provider_token_count(
|
|||
model_to_use=model_to_use or "",
|
||||
messages=messages,
|
||||
contents=contents,
|
||||
deployment=_with_resolved_credentials(deployment_in_region),
|
||||
deployment=await _with_resolved_credentials(deployment_in_region),
|
||||
request_model=request_model,
|
||||
tools=tools,
|
||||
system=system,
|
||||
|
|
@ -14830,7 +14829,7 @@ async def _try_native_responses_count(
|
|||
model_to_use=model_to_use,
|
||||
request=responses_request,
|
||||
policy=policy,
|
||||
deployment=_with_resolved_credentials(deployment),
|
||||
deployment=await _with_resolved_credentials(deployment),
|
||||
request_model=request_model,
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from litellm.types.realtime import (
|
|||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
|
||||
from litellm.utils import ProviderConfigManager, aresolve_api_key_secret_reference
|
||||
|
||||
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
|
||||
from ..litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
|
|
@ -810,7 +810,7 @@ async def realtime_health_check(
|
|||
resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints
|
||||
str | None, api_base or resolved_params.get("api_base")
|
||||
)
|
||||
resolved_api_key: Final = resolve_api_key_secret_reference(
|
||||
resolved_api_key: Final = await aresolve_api_key_secret_reference(
|
||||
cast( # cast-ok: provider parameters expose optional string credentials
|
||||
str | None, api_key or resolved_params.get("api_key")
|
||||
),
|
||||
|
|
|
|||
|
|
@ -443,6 +443,7 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
Rules,
|
||||
TextCompletionResponse,
|
||||
aresolve_api_key_secret_reference,
|
||||
function_setup,
|
||||
get_llm_provider,
|
||||
get_non_default_completion_params,
|
||||
|
|
@ -453,7 +454,6 @@ from litellm.utils import (
|
|||
lookup_model_info_for_registration,
|
||||
model_info_with_pricing_overrides,
|
||||
provider_rejectable_params,
|
||||
resolve_api_key_secret_reference,
|
||||
set_live_deployment_replay,
|
||||
)
|
||||
|
||||
|
|
@ -11882,7 +11882,7 @@ class Router:
|
|||
return
|
||||
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
|
||||
return
|
||||
api_key: Final = resolve_api_key_secret_reference(
|
||||
api_key: Final = await aresolve_api_key_secret_reference(
|
||||
params.api_key or dynamic_api_key,
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import traceback
|
|||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -167,12 +167,36 @@ def get_secret_str(
|
|||
|
||||
|
||||
LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/")
|
||||
GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/"
|
||||
API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX)
|
||||
_GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal"
|
||||
_GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS: Final = "3"
|
||||
_GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token"
|
||||
|
||||
|
||||
def _google_metadata_identity_url() -> str:
|
||||
host: Final = os.getenv("GCE_METADATA_HOST") or _GOOGLE_METADATA_DEFAULT_HOST
|
||||
return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity"
|
||||
|
||||
|
||||
def _google_metadata_timeout() -> float:
|
||||
return float(os.getenv("GCE_METADATA_TIMEOUT") or _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
class GoogleMetadataStatusError(ValueError):
|
||||
def __init__(self, status_code: int) -> None:
|
||||
super().__init__(f"Google OIDC provider failed with HTTP {status_code}")
|
||||
self.status_code: Final = status_code
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnresolvableSecretReference:
|
||||
kind: str
|
||||
reason: str
|
||||
retryable: bool = False
|
||||
|
||||
|
||||
OidcResolver: TypeAlias = Callable[[str], "str | UnresolvableSecretReference"]
|
||||
|
||||
|
||||
_OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"})
|
||||
|
|
@ -198,11 +222,52 @@ def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReferenc
|
|||
return secret
|
||||
|
||||
|
||||
def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference:
|
||||
def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretReference:
|
||||
if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX):
|
||||
return resolve_local_oidc_reference(value)
|
||||
try:
|
||||
token: Final = get_secret_str(value)
|
||||
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
|
||||
return _logged_unresolvable(
|
||||
value,
|
||||
"oidc/google",
|
||||
_GOOGLE_TOKEN_UNAVAILABLE,
|
||||
f"{type(exc).__name__}: {exc}",
|
||||
retryable=_metadata_server_failed_to_answer(exc),
|
||||
)
|
||||
if not token:
|
||||
return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value")
|
||||
return token
|
||||
|
||||
|
||||
def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool:
|
||||
match exc:
|
||||
case httpx.TimeoutException() | httpx.NetworkError():
|
||||
return True
|
||||
case GoogleMetadataStatusError(status_code=status_code):
|
||||
return status_code >= 500
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def cached_oidc_token(reference: str) -> str | None:
|
||||
cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped
|
||||
return cached if isinstance(cached, str) and cached else None
|
||||
|
||||
|
||||
def _logged_unresolvable(
|
||||
reference: str, kind: str, reason: str, detail: str | None = None, retryable: bool = False
|
||||
) -> UnresolvableSecretReference:
|
||||
verbose_logger.warning(
|
||||
"Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason
|
||||
"Secret reference %s could not be resolved: %s",
|
||||
_single_line(reference),
|
||||
reason if detail is None else f"{reason} ({_single_line(detail)})",
|
||||
)
|
||||
return UnresolvableSecretReference(kind=kind, reason=reason)
|
||||
return UnresolvableSecretReference(kind=kind, reason=reason, retryable=retryable)
|
||||
|
||||
|
||||
def _single_line(text: str) -> str:
|
||||
return text.replace("\r", "").replace("\n", "")
|
||||
|
||||
|
||||
def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str:
|
||||
|
|
@ -280,24 +345,29 @@ def get_secret(
|
|||
oidc_client = _get_oidc_http_handler()
|
||||
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
|
||||
response = oidc_client.get(
|
||||
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity",
|
||||
_google_metadata_identity_url(),
|
||||
params={"audience": oidc_aud},
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
timeout=_google_metadata_timeout(),
|
||||
)
|
||||
if response.status_code == 200:
|
||||
oidc_token = response.text
|
||||
ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60)
|
||||
if ttl > 0:
|
||||
if not oidc_token:
|
||||
verbose_logger.warning(
|
||||
"Google OIDC token for %s is empty; not caching it", _single_line(secret_name)
|
||||
)
|
||||
elif ttl > 0:
|
||||
oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl)
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
|
||||
secret_name,
|
||||
_single_line(secret_name),
|
||||
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
|
||||
)
|
||||
return oidc_token
|
||||
else:
|
||||
raise ValueError("Google OIDC provider failed")
|
||||
raise GoogleMetadataStatusError(response.status_code)
|
||||
elif oidc_provider == "circleci":
|
||||
# https://circleci.com/docs/openid-connect-tokens/
|
||||
env_secret = os.getenv("CIRCLE_OIDC_TOKEN")
|
||||
|
|
|
|||
|
|
@ -28,9 +28,10 @@ import textwrap
|
|||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache, wraps
|
||||
from functools import lru_cache, partial, wraps
|
||||
from importlib import resources
|
||||
from inspect import iscoroutine
|
||||
from io import StringIO
|
||||
|
|
@ -476,11 +477,14 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
|
|||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig
|
||||
from litellm.secret_managers.main import (
|
||||
LOCAL_OIDC_SECRET_PREFIXES,
|
||||
API_KEY_OIDC_SECRET_PREFIXES,
|
||||
GOOGLE_OIDC_SECRET_PREFIX,
|
||||
OidcResolver,
|
||||
UnresolvableSecretReference,
|
||||
cached_oidc_token,
|
||||
get_secret,
|
||||
oidc_reference_kind,
|
||||
resolve_local_oidc_reference,
|
||||
resolve_api_key_oidc_reference,
|
||||
)
|
||||
|
||||
from ._logging import is_debugging_on, verbose_logger
|
||||
|
|
@ -777,19 +781,56 @@ def resolve_api_key_secret_reference(
|
|||
return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream)
|
||||
|
||||
|
||||
async def aresolve_api_key_secret_reference(
|
||||
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
|
||||
) -> str | None:
|
||||
if not isinstance(api_key, AdminApiKeySecretReference):
|
||||
return api_key
|
||||
return await _resolved_off_the_event_loop(
|
||||
partial(_secret_behind_api_key_reference, api_key, model, llm_provider, upstream), api_key
|
||||
)
|
||||
|
||||
|
||||
_OIDC_TOKEN_FETCH_EXECUTOR: Final = ThreadPoolExecutor(thread_name_prefix="litellm-oidc-token-fetch")
|
||||
_Resolved: Final = TypeVar("_Resolved")
|
||||
|
||||
|
||||
async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolved], api_key: object) -> _Resolved:
|
||||
if not isinstance(api_key, AdminApiKeySecretReference) or not api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX):
|
||||
return resolve(resolve_api_key_oidc_reference)
|
||||
match cached_oidc_token(api_key):
|
||||
case str() as token:
|
||||
return resolve(lambda _reference: token)
|
||||
case None:
|
||||
caller_context: Final = contextvars.copy_context()
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference)
|
||||
)
|
||||
|
||||
|
||||
def _secret_behind_api_key_reference(
|
||||
reference: AdminApiKeySecretReference, model: object, llm_provider: object, upstream: Mapping[str, object]
|
||||
reference: AdminApiKeySecretReference,
|
||||
model: object,
|
||||
llm_provider: object,
|
||||
upstream: Mapping[str, object],
|
||||
resolve_oidc: OidcResolver = resolve_api_key_oidc_reference,
|
||||
) -> str:
|
||||
if not reference.may_reach(upstream):
|
||||
raise _secret_reference_error(_unreachable_upstream_message(reference), llm_provider=llm_provider, model=model)
|
||||
if not reference.startswith(LOCAL_OIDC_SECRET_PREFIXES):
|
||||
if not reference.startswith(API_KEY_OIDC_SECRET_PREFIXES):
|
||||
raise _secret_reference_error(
|
||||
f"api_key is an {oidc_reference_kind(reference)} secret reference, a kind api_key does not support. "
|
||||
"Use oidc/file/<absolute path> or oidc/env/<VARIABLE_NAME>.",
|
||||
"Use oidc/file/<absolute path>, oidc/env/<VARIABLE_NAME> or oidc/google/<audience>.",
|
||||
llm_provider=llm_provider,
|
||||
model=model,
|
||||
)
|
||||
match resolve_local_oidc_reference(reference):
|
||||
match resolve_oidc(reference):
|
||||
case UnresolvableSecretReference(kind=kind, reason=reason, retryable=True):
|
||||
raise ServiceUnavailableError(
|
||||
message=f"api_key {kind} secret reference could not be resolved: {reason}",
|
||||
llm_provider=llm_provider,
|
||||
model=model,
|
||||
)
|
||||
case UnresolvableSecretReference(kind=kind, reason=reason):
|
||||
raise _secret_reference_error(
|
||||
f"api_key {kind} secret reference could not be resolved: {reason}",
|
||||
|
|
@ -826,16 +867,31 @@ def load_credentials_from_list(kwargs: dict[str, object]) -> None:
|
|||
Updates kwargs with the credentials if credential_name in kwarg, then swaps an admin-configured oidc/ api_key
|
||||
reference for the secret it points to, read now so a rotated token is used on the next call
|
||||
"""
|
||||
kwargs.update(_credential_updates(kwargs))
|
||||
kwargs.update(_credential_updates(kwargs, _named_credential_values(kwargs)))
|
||||
|
||||
|
||||
def _credential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
async def acredential_updates(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
credential_values: Final = _named_credential_values(kwargs)
|
||||
api_key: Final = kwargs.get("api_key", credential_values.get("api_key"))
|
||||
return await _resolved_off_the_event_loop(
|
||||
partial(_credential_updates, kwargs, credential_values), _api_key_after_credentials(kwargs, credential_values)
|
||||
)
|
||||
|
||||
|
||||
def _api_key_after_credentials(kwargs: Mapping[str, object], credential_values: Mapping[str, object]) -> object:
|
||||
return kwargs.get("api_key", credential_values.get("api_key"))
|
||||
|
||||
|
||||
def _credential_updates(
|
||||
kwargs: Mapping[str, object],
|
||||
credential_values: Mapping[str, object],
|
||||
resolve_oidc: OidcResolver = resolve_api_key_oidc_reference,
|
||||
) -> Mapping[str, object]:
|
||||
api_key: Final = _api_key_after_credentials(kwargs, credential_values)
|
||||
if not isinstance(api_key, AdminApiKeySecretReference):
|
||||
return credential_values
|
||||
secret: Final = _secret_behind_api_key_reference(
|
||||
api_key,
|
||||
resolve_oidc=resolve_oidc,
|
||||
model=kwargs.get("model"),
|
||||
llm_provider=kwargs.get("custom_llm_provider"),
|
||||
upstream=MappingProxyType(
|
||||
|
|
@ -2216,7 +2272,7 @@ def client(original_function):
|
|||
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
## LOAD CREDENTIALS
|
||||
load_credentials_from_list(kwargs)
|
||||
kwargs.update(await acredential_updates(kwargs)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is the untyped request dict
|
||||
from litellm.llms.github_copilot.per_user_auth import aattach_github_copilot_user_session
|
||||
|
||||
await aattach_github_copilot_user_session(kwargs) # pyright: ignore[reportUnknownArgumentType] # kwargs is the untyped request dict
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -46,7 +48,17 @@ _SEARCH_PATH: Final = f"/v1/vector_stores/{_VECTOR_STORE}/search"
|
|||
_BATCH_DEPLOYMENT: Final = "wif-vllm-batch"
|
||||
_BEDROCK_MODEL: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5"
|
||||
_PROVIDER_INPUT_TOKENS: Final = 7919
|
||||
_UNSUPPORTED_REFERENCE: Final = "oidc/google/https://example.invalid"
|
||||
_UNSUPPORTED_REFERENCE: Final = "oidc/github/https://example.invalid"
|
||||
_GOOGLE_MODEL: Final = "hosted_vllm/wif-google"
|
||||
_GOOGLE_AUDIENCES: Final = MappingProxyType(
|
||||
{
|
||||
"config_inline": "https://vllm-inline.example.test",
|
||||
"config_credential": "https://vllm-credential.example.test",
|
||||
"db_credential": "https://vllm-db-credential.example.test",
|
||||
}
|
||||
)
|
||||
_UNAVAILABLE_AUDIENCE: Final = "https://vllm-unavailable.example.test"
|
||||
_IDENTITY_PATH: Final = "/computeMetadata/v1/instance/service-accounts/default/identity"
|
||||
_RUN_SECRETS_TOKEN: Final = "/run/secrets/wif-absent-token"
|
||||
_SOURCES: Final = ("config_credential", "config_inline", "db_credential", "db_inline")
|
||||
_UNREADABLE: Final = MappingProxyType(
|
||||
|
|
@ -108,6 +120,43 @@ def _reference(path: Path) -> str:
|
|||
return f"oidc/file/{path}"
|
||||
|
||||
|
||||
def _google_reference(audience: str) -> str:
|
||||
return f"oidc/google/{audience}"
|
||||
|
||||
|
||||
def _jwt_segment(part: Mapping[str, JsonValue]) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Google:
|
||||
expires: int
|
||||
|
||||
def token(self, audience: str) -> str:
|
||||
claims: Final = _jwt_segment({"iss": "https://accounts.google.com", "aud": audience, "exp": self.expires})
|
||||
return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{claims}.signature"
|
||||
|
||||
def bearers(self) -> frozenset[str]:
|
||||
return frozenset(f"Bearer {self.token(audience)}" for audience in _GOOGLE_AUDIENCES.values())
|
||||
|
||||
|
||||
def _metadata_server(google: _Google) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
target: Final = httpx.URL(request.target)
|
||||
audience: Final = target.params.get("audience")
|
||||
if audience is None or (request.method, target.path, request.headers.get("metadata-flavor")) != (
|
||||
"GET",
|
||||
_IDENTITY_PATH,
|
||||
"Google",
|
||||
):
|
||||
return Reply(status=404, body=b"not found", content_type="text/plain")
|
||||
if audience == _UNAVAILABLE_AUDIENCE:
|
||||
return Reply(status=503, body=b"metadata server unavailable", content_type="text/plain")
|
||||
return Reply(body=google.token(audience).encode(), content_type="text/plain")
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _secrets(directory: Path) -> _Secrets:
|
||||
allowed: Final = directory / "secrets"
|
||||
outside_dir: Final = directory / "outside"
|
||||
|
|
@ -169,9 +218,9 @@ def _served(request: Request) -> Reply:
|
|||
return openai_error(404)
|
||||
|
||||
|
||||
def _provider(secrets: _Secrets) -> Callable[[Request], Reply]:
|
||||
def _provider(secrets: _Secrets, google: _Google) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.headers.get("authorization") != f"Bearer {secrets.current()}":
|
||||
if request.headers.get("authorization") not in {f"Bearer {secrets.current()}", *google.bearers()}:
|
||||
return openai_error(401)
|
||||
return _served(request)
|
||||
|
||||
|
|
@ -224,6 +273,19 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path:
|
|||
"aws_bedrock_runtime_endpoint": provider.removesuffix("/v1"),
|
||||
},
|
||||
),
|
||||
_deployment(
|
||||
"wif-google-inline",
|
||||
{
|
||||
"model": _GOOGLE_MODEL,
|
||||
"api_key": _google_reference(_GOOGLE_AUDIENCES["config_inline"]),
|
||||
"api_base": provider,
|
||||
},
|
||||
),
|
||||
_deployment("wif-google-credential", _named(_GOOGLE_MODEL, "wif-google")),
|
||||
_deployment(
|
||||
"wif-google-unavailable",
|
||||
{"model": _GOOGLE_MODEL, "api_key": _google_reference(_UNAVAILABLE_AUDIENCE), "api_base": provider},
|
||||
),
|
||||
_deployment(
|
||||
"wif-mistral-passthrough",
|
||||
{
|
||||
|
|
@ -239,6 +301,7 @@ def _config(directory: Path, secrets: _Secrets, provider: str) -> Path:
|
|||
_credential("wif-outside", _reference(secrets.outside), provider),
|
||||
_credential("wif-unsupported", _UNSUPPORTED_REFERENCE, provider),
|
||||
_credential("wif-run-secrets", f"oidc/file/{_RUN_SECRETS_TOKEN}", provider),
|
||||
_credential("wif-google", _google_reference(_GOOGLE_AUDIENCES["config_credential"]), provider),
|
||||
],
|
||||
"vector_store_registry": [
|
||||
{
|
||||
|
|
@ -312,6 +375,8 @@ class _Rig:
|
|||
owned: OwnedProxy
|
||||
provider: Recorder
|
||||
intruder: Recorder
|
||||
metadata: Recorder
|
||||
google: _Google
|
||||
provider_base: str
|
||||
intruder_base: str
|
||||
secrets: _Secrets
|
||||
|
|
@ -339,17 +404,23 @@ def _is_discovery(request: Request) -> bool:
|
|||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("api_key_secret_reference").resolve()
|
||||
secrets: Final = _secrets(directory)
|
||||
google: Final = _Google(expires=int(time.time()) + 3600)
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
wire_server(_provider(secrets)) as provider_wire,
|
||||
wire_server(_provider(secrets, google)) as provider_wire,
|
||||
wire_server(_intruder) as intruder_wire,
|
||||
wire_server(_metadata_server(google)) as metadata_wire,
|
||||
):
|
||||
provider: Final = Recorder(provider_wire)
|
||||
provider_base: Final = f"{provider_wire.url}/v1"
|
||||
with owned_proxy_process(
|
||||
gateway,
|
||||
directory,
|
||||
{**secrets.overrides(), "MISTRAL_API_BASE": provider_wire.url},
|
||||
{
|
||||
**secrets.overrides(),
|
||||
"MISTRAL_API_BASE": provider_wire.url,
|
||||
"GCE_METADATA_HOST": httpx.URL(metadata_wire.url).netloc.decode(),
|
||||
},
|
||||
config=_config(directory, secrets, provider_base),
|
||||
workers=2,
|
||||
healthcheck=WorkerHealthcheck.LENIENT,
|
||||
|
|
@ -361,6 +432,8 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
|||
owned=owned,
|
||||
provider=provider,
|
||||
intruder=Recorder(intruder_wire),
|
||||
metadata=Recorder(metadata_wire),
|
||||
google=google,
|
||||
provider_base=provider_base,
|
||||
intruder_base=f"{intruder_wire.url}/v1",
|
||||
secrets=secrets,
|
||||
|
|
@ -708,7 +781,70 @@ def test_team_row_naming_the_credential_sends_the_token_only_to_its_host(rig: _R
|
|||
|
||||
def test_startup_discovery_sends_the_boot_token(rig: _Rig) -> None:
|
||||
sent: Final = frozenset(_bearers(tuple(request for request in rig.boot if _is_discovery(request))))
|
||||
assert sent == {f"Bearer {rig.boot_token}"}, sent
|
||||
assert sent - rig.google.bearers() == {f"Bearer {rig.boot_token}"}, sent
|
||||
|
||||
|
||||
def _discovery_bearers(rig: _Rig) -> frozenset[str | None]:
|
||||
return frozenset(_bearers(tuple(request for request in rig.provider.requests() if _is_discovery(request))))
|
||||
|
||||
|
||||
def test_startup_discovery_sends_the_google_token(rig: _Rig) -> None:
|
||||
expected: Final = frozenset(
|
||||
f"Bearer {rig.google.token(_GOOGLE_AUDIENCES[source])}" for source in ("config_inline", "config_credential")
|
||||
)
|
||||
sent: Final = eventually(
|
||||
partial(_discovery_bearers, rig), lambda seen: expected <= seen, seconds=30, return_last_on_timeout=True
|
||||
)
|
||||
assert expected <= sent, sent
|
||||
|
||||
|
||||
def _google_model(rig: _Rig, scenario: Scenario, source: str) -> str:
|
||||
match source:
|
||||
case "config_inline":
|
||||
return "wif-google-inline"
|
||||
case "config_credential":
|
||||
return "wif-google-credential"
|
||||
case "db_credential":
|
||||
credential: Final = f"db-google-{uuid.uuid4().hex}"
|
||||
rig.owned.gateway.post(
|
||||
"/credentials",
|
||||
_credential(credential, _google_reference(_GOOGLE_AUDIENCES[source]), rig.provider_base),
|
||||
)
|
||||
scenario.cleanups.callback(_delete_credential, rig.owned.gateway, credential)
|
||||
name, identity = _created_model(
|
||||
rig.owned.gateway,
|
||||
scenario,
|
||||
_deployment(f"db-google-{uuid.uuid4().hex}", _named(_GOOGLE_MODEL, credential)),
|
||||
)
|
||||
eventually(partial(rig.owned.serves_only_on_every_worker, name, identity), bool, seconds=120)
|
||||
return name
|
||||
case _:
|
||||
pytest.fail(f"unknown source {source}")
|
||||
|
||||
|
||||
def _asked_audiences(rig: _Rig) -> frozenset[str | None]:
|
||||
return frozenset(httpx.URL(request.target).params.get("audience") for request in rig.metadata.requests())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", tuple(_GOOGLE_AUDIENCES))
|
||||
def test_google_reference_sends_the_metadata_server_token_for_its_audience(rig: _Rig, source: str) -> None:
|
||||
audience: Final = _GOOGLE_AUDIENCES[source]
|
||||
with rig.owned.gateway.scenario() as scenario:
|
||||
marker: Final = _marker()
|
||||
response: Final = _chat(rig.owned.gateway, _google_model(rig, scenario, source), marker)
|
||||
sent: Final = _bearers(_carrying(rig.provider.requests(), marker))
|
||||
assert (response.status_code, sent) == (200, (f"Bearer {rig.google.token(audience)}",)), response.text
|
||||
assert audience in _asked_audiences(rig)
|
||||
|
||||
|
||||
def test_google_reference_fails_closed_when_the_metadata_server_fails(rig: _Rig) -> None:
|
||||
marker: Final = _marker()
|
||||
response: Final = _chat(rig.owned.gateway, "wif-google-unavailable", marker)
|
||||
reached: Final = _carrying(rig.provider.requests(), marker)
|
||||
assert (response.status_code, reached) == (503, ()), (response.text, _bearers(reached))
|
||||
assert "oidc/google secret reference could not be resolved" in response.text, response.text
|
||||
assert _UNAVAILABLE_AUDIENCE.removeprefix("https://") not in response.text, response.text
|
||||
assert _UNAVAILABLE_AUDIENCE in _asked_audiences(rig)
|
||||
|
||||
|
||||
def _side_call(rig: _Rig, seam: str, marker: str) -> tuple[httpx.Response, Callable[[Request], bool]]:
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from typing import Final
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import redact_internal_details_from_client_message
|
||||
|
|
@ -32,7 +34,7 @@ class MockHTTPHandler:
|
|||
self.text = "mocked_token"
|
||||
self.json_data = {"value": "mocked_token"}
|
||||
|
||||
def get(self, url, params=None, headers=None):
|
||||
def get(self, url, params=None, headers=None, timeout=None):
|
||||
# Store params for audience verification
|
||||
self.last_params = params
|
||||
logger.debug(
|
||||
|
|
@ -224,6 +226,23 @@ def test_oidc_google_failure():
|
|||
get_secret(secret_name)
|
||||
|
||||
|
||||
|
||||
def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_names(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GCE_METADATA_HOST", "127.0.0.1:8930")
|
||||
audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}"
|
||||
|
||||
with respx.mock(assert_all_called=True) as metadata:
|
||||
metadata.get(
|
||||
"http://127.0.0.1:8930/computeMetadata/v1/instance/service-accounts/default/identity",
|
||||
params={"audience": audience},
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
).respond(200, text="metadata-host-token")
|
||||
token: Final = get_secret(f"oidc/google/{audience}")
|
||||
|
||||
assert token == "metadata-host-token"
|
||||
|
||||
def test_oidc_circleci_success(monkeypatch):
|
||||
monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token")
|
||||
|
||||
|
|
|
|||
|
|
@ -5,14 +5,19 @@ import copy
|
|||
import contextlib
|
||||
import contextvars
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Final, SupportsIndex, cast
|
||||
from unittest import mock
|
||||
|
|
@ -79,8 +84,10 @@ from litellm.types.utils import (
|
|||
all_litellm_params,
|
||||
bedrock_batch_litellm_params,
|
||||
)
|
||||
from litellm.secret_managers.main import oidc_cache
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import (
|
||||
acredential_updates,
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
add_custom_logger_callback_to_specific_event,
|
||||
check_valid_key,
|
||||
|
|
@ -7318,7 +7325,6 @@ def test_unresolvable_oidc_api_key_fails_closed_naming_the_reason_without_its_ta
|
|||
("reference", "kind"),
|
||||
[
|
||||
("oidc/env_path/MISTRAL_SA_TOKEN_PATH", "oidc/env_path"),
|
||||
("oidc/google/https://api.mistral.ai", "oidc/google"),
|
||||
("oidc/github/mistral", "oidc/github"),
|
||||
("oidc/env_MISTRAL_SA_TOKEN_PATH", "oidc/<unknown kind>"),
|
||||
("oidc/LIT-UNKNOWN-KIND/MISTRAL_SA_TOKEN_PATH", "oidc/<unknown kind>"),
|
||||
|
|
@ -7339,6 +7345,7 @@ def test_oidc_api_key_of_a_kind_api_key_cannot_resolve_fails_closed_as_an_authen
|
|||
assert not route.called
|
||||
assert excinfo.value.status_code == 401
|
||||
assert f"{kind} secret reference" in excinfo.value.message
|
||||
assert "oidc/google/<audience>" in excinfo.value.message
|
||||
assert reference.removeprefix(kind).lstrip("/") not in excinfo.value.message
|
||||
assert "MISTRAL_SA_TOKEN_PATH" not in excinfo.value.message
|
||||
|
||||
|
|
@ -7354,7 +7361,7 @@ async def test_router_fails_over_from_a_deployment_whose_oidc_api_key_kind_canno
|
|||
"model_name": "devstral",
|
||||
"litellm_params": {
|
||||
"model": "mistral/devstral-latest",
|
||||
"api_key": "oidc/google/https://api.mistral.ai",
|
||||
"api_key": "oidc/github/mistral",
|
||||
"order": 1,
|
||||
},
|
||||
},
|
||||
|
|
@ -7393,6 +7400,219 @@ def test_an_oidc_api_key_sent_with_the_call_goes_upstream_as_written(
|
|||
assert sent == f"Bearer {reference}"
|
||||
|
||||
|
||||
_GOOGLE_METADATA_IDENTITY: Final = (
|
||||
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity"
|
||||
)
|
||||
_VLLM_BASE: Final = "https://vllm.example.test/v1"
|
||||
_VLLM_CHAT_URL: Final = f"{_VLLM_BASE}/chat/completions"
|
||||
_VLLM_MODEL: Final = "hosted_vllm/qwen3-0.6b"
|
||||
_VLLM_REPLY: Final = {**_MISTRAL_REPLY, "model": "qwen3-0.6b"}
|
||||
_CLOCK_START: Final = 1_900_000_000.0
|
||||
_GOOGLE_TOKEN_LIFETIME: Final = 3600
|
||||
|
||||
|
||||
def _jwt_segment(claims: Mapping[str, object]) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
def _google_id_token(audience: str, issued_at: float) -> str:
|
||||
claims: Final = {"aud": audience, "iat": int(issued_at), "exp": int(issued_at) + _GOOGLE_TOKEN_LIFETIME}
|
||||
return f"{_jwt_segment({'alg': 'RS256', 'typ': 'JWT'})}.{_jwt_segment(claims)}.signature"
|
||||
|
||||
|
||||
def _google_reference() -> tuple[str, str]:
|
||||
audience: Final = f"https://vllm.example.test/{uuid.uuid4().hex}"
|
||||
return audience, f"oidc/google/{audience}"
|
||||
|
||||
|
||||
def _metadata_identity(fetched_on: queue.SimpleQueue[int], request: httpx.Request) -> httpx.Response:
|
||||
fetched_on.put(threading.get_ident())
|
||||
if request.headers.get("Metadata-Flavor") != "Google":
|
||||
return httpx.Response(403)
|
||||
return httpx.Response(200, text=_google_id_token(request.url.params["audience"], time.time()))
|
||||
|
||||
|
||||
def _vllm_credential(api_key: str) -> CredentialItem:
|
||||
return CredentialItem(
|
||||
credential_name="vllm-sa", credential_values={"api_key": api_key, "api_base": _VLLM_BASE}, credential_info={}
|
||||
)
|
||||
|
||||
|
||||
async def _ping_vllm_through(call_path: str, credential_name: str) -> None:
|
||||
if call_path == "async":
|
||||
await litellm.acompletion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name)
|
||||
return
|
||||
litellm.completion(model=_VLLM_MODEL, messages=_PING, litellm_credential_name=credential_name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("call_path", ["sync", "async"])
|
||||
async def test_named_credential_oidc_google_api_key_sends_the_metadata_token_and_refreshes_it_before_it_expires(
|
||||
monkeypatch: pytest.MonkeyPatch, call_path: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
|
||||
audience, reference = _google_reference()
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
refreshed_at: Final = _CLOCK_START + _GOOGLE_TOKEN_LIFETIME - 59
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock(
|
||||
side_effect=partial(_metadata_identity, queue.SimpleQueue())
|
||||
)
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
await _ping_vllm_through(call_path, "vllm-sa")
|
||||
await _ping_vllm_through(call_path, "vllm-sa")
|
||||
fetches_within_lifetime: Final = metadata.call_count
|
||||
monkeypatch.setattr(time, "time", lambda: refreshed_at)
|
||||
await _ping_vllm_through(call_path, "vllm-sa")
|
||||
sent: Final = [call.request.headers["Authorization"] for call in chat.calls]
|
||||
audiences: Final = [call.request.url.params["audience"] for call in metadata.calls]
|
||||
|
||||
first_token: Final = f"Bearer {_google_id_token(audience, _CLOCK_START)}"
|
||||
assert sent == [first_token, first_token, f"Bearer {_google_id_token(audience, refreshed_at)}"]
|
||||
assert (fetches_within_lifetime, audiences) == (1, [audience, audience])
|
||||
assert litellm.credential_list[0].credential_values["api_key"] == reference
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_and_keeps_the_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
|
||||
audience, reference = _google_reference()
|
||||
fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vllm",
|
||||
"litellm_params": {"model": _VLLM_MODEL, "api_base": _VLLM_BASE, "api_key": reference},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
try:
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on))
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
await router.acompletion(model="vllm", messages=_PING)
|
||||
sent: Final = chat.calls.last.request.headers["Authorization"]
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
assert sent == f"Bearer {_google_id_token(audience, _CLOCK_START)}"
|
||||
assert fetched_on.qsize() == 1
|
||||
assert fetched_on.get_nowait() != threading.get_ident()
|
||||
assert router.get_model_list(model_name="vllm")[0]["litellm_params"]["api_key"] == reference
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("call_path", ["sync", "async"])
|
||||
@pytest.mark.parametrize(
|
||||
("metadata_failure", "status_code"),
|
||||
[
|
||||
(lambda: httpx.Response(500, text="metadata server unavailable"), 503),
|
||||
(lambda: httpx.Response(403, text="Missing Metadata-Flavor:Google header"), 401),
|
||||
(lambda: httpx.Response(200, text=""), 401),
|
||||
(lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503),
|
||||
(lambda: httpx.ReadTimeout("timed out"), 503),
|
||||
],
|
||||
ids=["server error", "refused", "empty token", "not on GCP", "timeout"],
|
||||
)
|
||||
async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
call_path: str,
|
||||
metadata_failure: Callable[[], httpx.Response | httpx.HTTPError],
|
||||
status_code: int,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
|
||||
audience, reference = _google_reference()
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
recovered_token: Final = _google_id_token(audience, _CLOCK_START)
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(
|
||||
side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)]
|
||||
)
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises(
|
||||
(litellm.AuthenticationError, litellm.ServiceUnavailableError)
|
||||
) as excinfo:
|
||||
await _ping_vllm_through(call_path, "vllm-sa")
|
||||
sent_while_failing: Final = chat.call_count
|
||||
await _ping_vllm_through(call_path, "vllm-sa")
|
||||
sent_after_recovery: Final = [call.request.headers["Authorization"] for call in chat.calls]
|
||||
|
||||
assert excinfo.value.status_code == status_code
|
||||
assert "api_key oidc/google secret reference could not be resolved" in excinfo.value.message
|
||||
assert audience.removeprefix("https://") not in excinfo.value.message
|
||||
assert reference in caplog.text
|
||||
assert sent_while_failing == 0
|
||||
assert sent_after_recovery == [f"Bearer {recovered_token}"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_oidc_google_api_key_token_cached_when_the_call_starts_is_sent_though_it_expires_mid_call(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
|
||||
audience, reference = _google_reference()
|
||||
cached_token: Final = _google_id_token(audience, _CLOCK_START)
|
||||
oidc_cache.set_cache(key=reference, value=cached_token, ttl=60)
|
||||
clock: Final = itertools.chain([_CLOCK_START + 60], itertools.repeat(_CLOCK_START + 61))
|
||||
monkeypatch.setattr(time, "time", lambda: next(clock))
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
fetched_on: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=partial(_metadata_identity, fetched_on))
|
||||
updates: Final = await acredential_updates({"model": _VLLM_MODEL, "litellm_credential_name": "vllm-sa"})
|
||||
|
||||
assert updates["api_key"] == cached_token
|
||||
assert fetched_on.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hung_metadata_server_fails_an_oidc_google_api_key_call_with_a_503_and_leaves_the_default_executor_free(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
_, reference = _google_reference()
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1")
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
loop.set_default_executor(ThreadPoolExecutor(max_workers=1))
|
||||
|
||||
with socket.create_server(("127.0.0.1", 0)) as never_answering, respx.mock(assert_all_called=False) as upstream:
|
||||
monkeypatch.setenv("GCE_METADATA_HOST", f"127.0.0.1:{never_answering.getsockname()[1]}")
|
||||
upstream.route(host="127.0.0.1").pass_through()
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
call: Final = asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa"))
|
||||
await asyncio.sleep(0.2)
|
||||
other_work: Final = await asyncio.wait_for(loop.run_in_executor(None, lambda: "done"), timeout=0.5)
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as excinfo:
|
||||
await asyncio.wait_for(call, timeout=10)
|
||||
|
||||
assert other_work == "done"
|
||||
assert excinfo.value.status_code == 503
|
||||
assert "the metadata server did not return an ID token" in excinfo.value.message
|
||||
assert chat.call_count == 0
|
||||
|
||||
|
||||
def test_an_oidc_google_api_key_sent_with_the_call_goes_upstream_as_written_without_a_metadata_fetch() -> None:
|
||||
_, reference = _google_reference()
|
||||
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
litellm.completion(model=_VLLM_MODEL, api_base=_VLLM_BASE, messages=_PING, api_key=reference)
|
||||
sent: Final = chat.calls.last.request.headers["Authorization"]
|
||||
|
||||
assert sent == f"Bearer {reference}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "expected"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue