mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(secret_managers): keep get_secret's metadata wait, share concurrent google token fetches, refuse oidc/google on pass-through
A model's api_key fetch keeps the 3 second GCE_METADATA_TIMEOUT wait while get_secret callers keep the handler default. Any metadata transport error answers a retryable 503. Async cache misses for one reference share one fetch, and a call the reference may not go to is refused before any fetch. Pass-through routes answer 401 for an oidc/google credential instead of sending the reference text. The metadata client is closed after each fetch, and the unused sync resolver is removed.
This commit is contained in:
parent
2087d8bd89
commit
efe52b79b5
6 changed files with 183 additions and 47 deletions
|
|
@ -11,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
|
|||
)
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.secret_managers.main import (
|
||||
GOOGLE_OIDC_SECRET_PREFIX,
|
||||
UnresolvableSecretReference,
|
||||
get_secret_str,
|
||||
resolve_local_oidc_reference,
|
||||
|
|
@ -46,6 +47,14 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str |
|
|||
def _resolve_oidc_reference(credential: str | None) -> str | None:
|
||||
if credential is None:
|
||||
return None
|
||||
if credential.startswith(GOOGLE_OIDC_SECRET_PREFIX):
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail=(
|
||||
"pass-through credential oidc/google secret reference could not be resolved: pass-through routes "
|
||||
"do not fetch Google ID tokens. Use a static key, oidc/file/<absolute path> or oidc/env/<VARIABLE_NAME>."
|
||||
),
|
||||
)
|
||||
match resolve_local_oidc_reference(credential):
|
||||
case UnresolvableSecretReference(kind=kind, reason=reason):
|
||||
raise HTTPException(
|
||||
|
|
|
|||
|
|
@ -228,7 +228,9 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere
|
|||
if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX):
|
||||
return resolve_local_oidc_reference(value)
|
||||
try:
|
||||
token: Final = get_secret_str(value)
|
||||
token: Final = _google_id_token(
|
||||
value, value.removeprefix(GOOGLE_OIDC_SECRET_PREFIX), timeout=_google_metadata_timeout()
|
||||
)
|
||||
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
|
||||
return _logged_unresolvable(
|
||||
value,
|
||||
|
|
@ -244,7 +246,7 @@ def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretRefere
|
|||
|
||||
def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool:
|
||||
match exc:
|
||||
case httpx.TimeoutException() | httpx.NetworkError():
|
||||
case httpx.TransportError():
|
||||
return True
|
||||
case GoogleMetadataStatusError(status_code=status_code):
|
||||
return status_code >= 500
|
||||
|
|
@ -252,6 +254,38 @@ def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL |
|
|||
return False
|
||||
|
||||
|
||||
def _google_id_token(reference: str, audience: str, timeout: float | None) -> str:
|
||||
cached: Final = cached_oidc_token(reference)
|
||||
if cached is not None:
|
||||
return cached
|
||||
oidc_client: Final = _get_oidc_http_handler()
|
||||
try:
|
||||
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
|
||||
response: Final = oidc_client.get(
|
||||
_google_metadata_identity_url(),
|
||||
params={"audience": audience},
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
timeout=timeout,
|
||||
)
|
||||
finally:
|
||||
oidc_client.close()
|
||||
if response.status_code != 200:
|
||||
raise GoogleMetadataStatusError(response.status_code)
|
||||
token: Final = response.text
|
||||
ttl: Final = _oidc_token_cache_ttl(token, 3600 - 60)
|
||||
if not token:
|
||||
verbose_logger.warning("Google OIDC token for %s is empty; not caching it", _single_line(reference))
|
||||
elif ttl > 0:
|
||||
oidc_cache.set_cache(key=reference, value=token, ttl=ttl)
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
|
||||
_single_line(reference),
|
||||
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def cached_oidc_token(reference: str) -> str | None:
|
||||
cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped
|
||||
return cached if isinstance(cached, str) and cached else None
|
||||
|
|
@ -340,36 +374,7 @@ def get_secret(
|
|||
oidc_aud = "/".join(secret_name_split.split("/")[1:])
|
||||
# TODO: Add caching for HTTP requests
|
||||
if oidc_provider == "google":
|
||||
oidc_token = oidc_cache.get_cache(key=secret_name)
|
||||
if oidc_token is not None:
|
||||
return oidc_token
|
||||
|
||||
oidc_client = _get_oidc_http_handler()
|
||||
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
|
||||
response = oidc_client.get(
|
||||
_google_metadata_identity_url(),
|
||||
params={"audience": oidc_aud},
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
timeout=_google_metadata_timeout(),
|
||||
)
|
||||
if response.status_code == 200:
|
||||
oidc_token = response.text
|
||||
ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60)
|
||||
if not oidc_token:
|
||||
verbose_logger.warning(
|
||||
"Google OIDC token for %s is empty; not caching it", _single_line(secret_name)
|
||||
)
|
||||
elif ttl > 0:
|
||||
oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl)
|
||||
else:
|
||||
verbose_logger.warning(
|
||||
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
|
||||
_single_line(secret_name),
|
||||
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
|
||||
)
|
||||
return oidc_token
|
||||
else:
|
||||
raise GoogleMetadataStatusError(response.status_code)
|
||||
return _google_id_token(secret_name, oidc_aud, timeout=None)
|
||||
elif oidc_provider == "circleci":
|
||||
# https://circleci.com/docs/openid-connect-tokens/
|
||||
env_secret = os.getenv("CIRCLE_OIDC_TOKEN")
|
||||
|
|
|
|||
|
|
@ -773,14 +773,6 @@ def _named_credential_values(kwargs: Mapping[str, object]) -> Mapping[str, objec
|
|||
)
|
||||
|
||||
|
||||
def resolve_api_key_secret_reference(
|
||||
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
|
||||
) -> str | None:
|
||||
if not isinstance(api_key, AdminApiKeySecretReference):
|
||||
return api_key
|
||||
return _secret_behind_api_key_reference(api_key, model=model, llm_provider=llm_provider, upstream=upstream)
|
||||
|
||||
|
||||
async def aresolve_api_key_secret_reference(
|
||||
api_key: str | None, model: object, llm_provider: object, upstream: Mapping[str, object]
|
||||
) -> str | None:
|
||||
|
|
@ -802,10 +794,40 @@ async def _resolved_off_the_event_loop(resolve: Callable[[OidcResolver], _Resolv
|
|||
case str() as token:
|
||||
return resolve(lambda _reference: token)
|
||||
case None:
|
||||
caller_context: Final = contextvars.copy_context()
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve, resolve_api_key_oidc_reference)
|
||||
_raise_if_the_reference_may_not_be_used(resolve)
|
||||
fetched: Final = await asyncio.shield(
|
||||
_google_token_fetch(asyncio.get_running_loop(), api_key).start_or_join()
|
||||
)
|
||||
return resolve(lambda _reference: fetched)
|
||||
|
||||
|
||||
def _raise_if_the_reference_may_not_be_used(resolve: Callable[[OidcResolver], object]) -> None:
|
||||
_ = resolve(lambda _reference: "")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _GoogleTokenFetch:
|
||||
reference: str
|
||||
in_flight: asyncio.Future[str | UnresolvableSecretReference] | None = None
|
||||
|
||||
def start_or_join(self) -> asyncio.Future[str | UnresolvableSecretReference]:
|
||||
if self.in_flight is not None:
|
||||
return self.in_flight
|
||||
caller_context: Final = contextvars.copy_context()
|
||||
fetch: Final = asyncio.get_running_loop().run_in_executor(
|
||||
_OIDC_TOKEN_FETCH_EXECUTOR, lambda: caller_context.run(resolve_api_key_oidc_reference, self.reference)
|
||||
)
|
||||
fetch.add_done_callback(self._land)
|
||||
self.in_flight = fetch
|
||||
return fetch
|
||||
|
||||
def _land(self, _fetch: asyncio.Future[str | UnresolvableSecretReference]) -> None:
|
||||
self.in_flight = None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _google_token_fetch(loop: asyncio.AbstractEventLoop, reference: str) -> _GoogleTokenFetch:
|
||||
return _GoogleTokenFetch(reference)
|
||||
|
||||
|
||||
def _secret_behind_api_key_reference(
|
||||
|
|
|
|||
|
|
@ -485,7 +485,7 @@ def test_oidc_env_path_reference_is_not_resolved(tmp_path, monkeypatch):
|
|||
)
|
||||
|
||||
|
||||
def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch):
|
||||
def test_oidc_google_reference_is_refused_without_a_metadata_fetch(monkeypatch):
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("network-backed oidc reference was resolved inside the request path")
|
||||
|
||||
|
|
@ -494,11 +494,24 @@ def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch):
|
|||
lambda name, *a, **k: "oidc/google/https://example.com" if name == "OPENAI_API_KEY" else fail(),
|
||||
)
|
||||
|
||||
assert (
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "oidc/google/https://example.com"
|
||||
|
||||
assert excinfo.value.status_code == 401
|
||||
assert "pass-through credential oidc/google" in str(excinfo.value.detail)
|
||||
assert "example.com" not in str(excinfo.value.detail)
|
||||
|
||||
|
||||
def test_deployment_oidc_google_reference_is_not_sent_upstream(monkeypatch):
|
||||
llm_router = litellm.Router(
|
||||
model_list=[_flagged_deployment("openai/gpt-4o", api_key="oidc/google/https://vllm.example.test")]
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_passthrough_router(llm_router).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
|
||||
assert excinfo.value.status_code == 401
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
async def _drain_logging_worker():
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ class MockHTTPHandler:
|
|||
mock_response.json.return_value = self.json_data
|
||||
return mock_response
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_oidc_cache():
|
||||
|
|
@ -248,6 +251,24 @@ def test_oidc_google_fetches_the_token_from_the_metadata_host_gce_metadata_host_
|
|||
|
||||
assert token == "metadata-host-token"
|
||||
|
||||
def test_oidc_google_secret_outside_a_model_api_key_keeps_its_metadata_wait_when_gce_metadata_timeout_is_set(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("GCE_METADATA_TIMEOUT", "1")
|
||||
audience: Final = f"https://sts.example.test/{uuid.uuid4().hex}"
|
||||
|
||||
with respx.mock(assert_all_called=True) as metadata:
|
||||
identity: Final = metadata.get(
|
||||
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity",
|
||||
params={"audience": audience},
|
||||
).respond(200, text="web-identity-token")
|
||||
token: Final = get_secret(f"oidc/google/{audience}")
|
||||
waits: Final = identity.calls.last.request.extensions["timeout"]
|
||||
|
||||
assert token == "web-identity-token"
|
||||
assert waits == {"connect": 5.0, "read": 600.0, "write": 600.0, "pool": 600.0}
|
||||
|
||||
|
||||
def test_oidc_circleci_success(monkeypatch):
|
||||
monkeypatch.setenv("CIRCLE_OIDC_TOKEN", "circleci_token")
|
||||
|
||||
|
|
|
|||
|
|
@ -7518,8 +7518,9 @@ async def test_router_fetches_an_oidc_google_api_key_token_off_the_event_loop_an
|
|||
(lambda: httpx.Response(200, text=""), 401),
|
||||
(lambda: httpx.ConnectError("[Errno -2] Name or service not known"), 503),
|
||||
(lambda: httpx.ReadTimeout("timed out"), 503),
|
||||
(lambda: httpx.RemoteProtocolError("Server disconnected without sending a response."), 503),
|
||||
],
|
||||
ids=["server error", "refused", "empty token", "not on GCP", "timeout"],
|
||||
ids=["server error", "refused", "empty token", "not on GCP", "timeout", "disconnected"],
|
||||
)
|
||||
async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_and_recovers_on_the_next_call(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -7610,6 +7611,71 @@ async def test_a_pending_oidc_google_metadata_fetch_leaves_the_default_executor_
|
|||
assert chat.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"metadata_answer",
|
||||
[lambda token: httpx.Response(200, text=token), lambda _token: httpx.Response(500, text="metadata server down")],
|
||||
ids=["token", "outage"],
|
||||
)
|
||||
async def test_concurrent_calls_that_miss_the_oidc_google_token_cache_share_one_metadata_fetch(
|
||||
monkeypatch: pytest.MonkeyPatch, metadata_answer: Callable[[str], httpx.Response]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(time, "time", lambda: _CLOCK_START)
|
||||
audience, reference = _google_reference()
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
token: Final = _google_id_token(audience, _CLOCK_START)
|
||||
fetch_started: Final[queue.SimpleQueue[int]] = queue.SimpleQueue()
|
||||
release: Final = threading.Event()
|
||||
|
||||
def held_metadata(_request: httpx.Request) -> httpx.Response:
|
||||
fetch_started.put(threading.get_ident())
|
||||
release.wait(timeout=30)
|
||||
return metadata_answer(token)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as observer, respx.mock(assert_all_called=False) as upstream:
|
||||
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).mock(side_effect=held_metadata)
|
||||
chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY)
|
||||
calls: Final = [asyncio.ensure_future(_ping_vllm_through("async", "vllm-sa")) for _ in range(24)]
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.get_running_loop().run_in_executor(observer, fetch_started.get), timeout=30)
|
||||
await asyncio.sleep(0.5)
|
||||
finally:
|
||||
release.set()
|
||||
outcomes: Final = await asyncio.gather(*calls, return_exceptions=True)
|
||||
sent: Final = [call.request.headers["Authorization"] for call in chat.calls]
|
||||
|
||||
assert metadata.call_count == 1
|
||||
if metadata.calls.last.response.status_code == 200:
|
||||
assert outcomes == [None] * 24
|
||||
assert sent == [f"Bearer {token}"] * 24
|
||||
else:
|
||||
assert [getattr(outcome, "status_code", None) for outcome in outcomes] == [503] * 24
|
||||
assert sent == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_call_an_oidc_google_api_key_may_not_go_to_is_refused_without_a_metadata_fetch(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_, reference = _google_reference()
|
||||
monkeypatch.setattr(litellm, "credential_list", [_vllm_credential(reference)])
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
metadata: Final = upstream.get(_GOOGLE_METADATA_IDENTITY).respond(500)
|
||||
with pytest.raises(litellm.AuthenticationError) as excinfo:
|
||||
await acredential_updates(
|
||||
{
|
||||
"model": _VLLM_MODEL,
|
||||
"litellm_credential_name": "vllm-sa",
|
||||
"api_base": "https://elsewhere.example.test/v1",
|
||||
}
|
||||
)
|
||||
|
||||
assert "only to that credential's own api_base" in excinfo.value.message
|
||||
assert metadata.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "bound"),
|
||||
[(None, 3.0), ("1.5", 1.5), ("three seconds", 3.0), ("0", 3.0), ("-2", 3.0)],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue