mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(proxy): resolve oidc/ pass-through credentials on every request (#44577)
* fix(proxy): resolve oidc/ pass-through credentials on every request A pass-through credential (a use_in_pass_through deployment's api_key or the provider's env var, e.g. TYPESAFE_API_KEY) was sent as a literal string, so an `oidc/...` reference such as `oidc/file//var/run/secrets/<name>/token` ended up on the wire as `Bearer oidc/file/...`, and a rotating projected Kubernetes service account token could not be used for pass-through auth. Resolve credentials that start with `oidc/` through get_secret_str() on every get_credentials() call, so the file is re-read and rotations apply without a restart. oidc/file/ keeps its credential-directory allowlist; other credentials are returned unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(proxy): only resolve local oidc/ pass-through credentials inline get_credentials() runs synchronously inside async pass-through routes, so resolving oidc/google/, oidc/github/ etc. on a cache miss would make a blocking HTTP call to the identity provider (timeout up to 600s) on the event loop. Limit per-request resolution to oidc/file/, oidc/env/ and oidc/env_path/, which need no network I/O; network-backed references are left unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(proxy): surface unreadable oidc/ pass-through credentials as 401 A missing or disallowed token file raised FileNotFoundError or OidcPathNotAllowedError out of the route, which the proxy turned into a generic 500 on every request. The resolver now raises an HTTPException 401 that names the credential reference and the reason, matching how the pass-through routes already report an unset key. oidc/env_path/ is no longer resolved inline because it reads any file named by an environment variable without the oidc/file/ credential directory allowlist; it is forwarded unchanged like the network-backed references. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: ryan-crabbe-berri <ryan@berri.ai>
This commit is contained in:
Sam Mobach
GitHub
parent
2cee61626d
commit
292fcb1c65
2 changed files with 124 additions and 39 deletions
|
|
@ -2,6 +2,8 @@ import json
|
|||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
|
||||
|
|
@ -37,6 +39,24 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str |
|
|||
return credentials
|
||||
|
||||
|
||||
# Network I/O prohibited; get_credentials() is synchronous inside async routes.
|
||||
# oidc/env_path/ is excluded because it bypasses the oidc/file/ credential-directory allowlist.
|
||||
_LOCAL_OIDC_PREFIXES: Final = ("oidc/file/", "oidc/env/")
|
||||
|
||||
|
||||
def _resolve_oidc_reference(credential: str | None) -> str | None:
|
||||
"""Resolve local ``oidc/...`` credentials on every call to pick up rotated tokens."""
|
||||
if credential is None or not credential.startswith(_LOCAL_OIDC_PREFIXES):
|
||||
return credential
|
||||
try:
|
||||
return get_secret_str(credential)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail=f"pass-through credential {credential} could not be resolved: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
class PassthroughEndpointRouter:
|
||||
"""
|
||||
Use this class to Get credentials for pass-through endpoints
|
||||
|
|
@ -60,14 +80,14 @@ class PassthroughEndpointRouter:
|
|||
region_name=region_name,
|
||||
)
|
||||
if deployment_api_key is not None:
|
||||
return deployment_api_key
|
||||
return _resolve_oidc_reference(deployment_api_key)
|
||||
verbose_router_logger.debug(
|
||||
"No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider
|
||||
)
|
||||
_env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return get_secret_str(_env_variable_name)
|
||||
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
|
||||
|
||||
def _get_deployment_api_key(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
|
|
@ -45,10 +46,7 @@ def test_credential_loaded_after_deployment_registration_still_resolves():
|
|||
|
||||
CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-loaded-after-boot")])
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "sk-loaded-after-boot"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-loaded-after-boot"
|
||||
|
||||
|
||||
def test_credential_rotation_is_reflected_without_deployment_update():
|
||||
|
|
@ -58,17 +56,11 @@ def test_credential_rotation_is_reflected_without_deployment_update():
|
|||
)
|
||||
passthrough_router = _passthrough_router(llm_router)
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "sk-before-rotation"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-before-rotation"
|
||||
|
||||
CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-after-rotation")])
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "sk-after-rotation"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-after-rotation"
|
||||
|
||||
|
||||
def test_deleted_deployment_stops_serving_its_key(monkeypatch):
|
||||
|
|
@ -80,9 +72,7 @@ def test_deleted_deployment_stops_serving_its_key(monkeypatch):
|
|||
llm_router.set_model_list([])
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
|
||||
|
||||
def test_inline_api_key_resolves_without_credential_name():
|
||||
|
|
@ -91,10 +81,7 @@ def test_inline_api_key_resolves_without_credential_name():
|
|||
)
|
||||
passthrough_router = _passthrough_router(llm_router)
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="anthropic", region_name=None)
|
||||
== "sk-ant-inline"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="anthropic", region_name=None) == "sk-ant-inline"
|
||||
|
||||
|
||||
def test_missing_credential_and_no_inline_key_falls_back_to_env(monkeypatch):
|
||||
|
|
@ -104,9 +91,7 @@ def test_missing_credential_and_no_inline_key_falls_back_to_env(monkeypatch):
|
|||
passthrough_router = _passthrough_router(llm_router)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
|
||||
|
||||
def test_deployment_for_other_provider_does_not_match():
|
||||
|
|
@ -147,9 +132,7 @@ def test_first_matching_deployment_wins():
|
|||
def test_assemblyai_region_matching():
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
_flagged_deployment(
|
||||
"assemblyai/best", api_key="sk-eu", api_base="https://api.eu.assemblyai.com"
|
||||
),
|
||||
_flagged_deployment("assemblyai/best", api_key="sk-eu", api_base="https://api.eu.assemblyai.com"),
|
||||
_flagged_deployment("assemblyai/best", api_key="sk-us", api_base="https://api.assemblyai.com"),
|
||||
]
|
||||
)
|
||||
|
|
@ -179,9 +162,7 @@ def test_env_fallback_when_no_router(monkeypatch):
|
|||
passthrough_router = _passthrough_router(None)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
|
||||
assert (
|
||||
passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
)
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-from-env"
|
||||
|
||||
|
||||
def test_returns_none_when_no_router_and_no_env():
|
||||
|
|
@ -216,9 +197,7 @@ def test_vertex_deployment_resolves_via_named_credential():
|
|||
)
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
_vertex_deployment(
|
||||
"gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gcp"
|
||||
)
|
||||
_vertex_deployment("gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gcp")
|
||||
]
|
||||
)
|
||||
passthrough_router = _passthrough_router(llm_router)
|
||||
|
|
@ -276,9 +255,7 @@ def test_vertex_model_hint_prefers_matching_deployment():
|
|||
passthrough_router = _passthrough_router(_two_vertex_deployments_router())
|
||||
|
||||
by_alias = passthrough_router.get_vertex_credentials_from_router_deployments(model="gemini-live")
|
||||
by_upstream_id = passthrough_router.get_vertex_credentials_from_router_deployments(
|
||||
model="gemini-live-2.5-flash"
|
||||
)
|
||||
by_upstream_id = passthrough_router.get_vertex_credentials_from_router_deployments(model="gemini-live-2.5-flash")
|
||||
|
||||
assert by_alias is not None and by_alias.vertex_project == "proj-live"
|
||||
assert by_upstream_id is not None and by_upstream_id.vertex_project == "proj-live"
|
||||
|
|
@ -388,12 +365,100 @@ def test_vertex_deployment_with_deleted_credential_is_skipped(monkeypatch):
|
|||
)
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
_vertex_deployment(
|
||||
"gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gone"
|
||||
)
|
||||
_vertex_deployment("gemini-live", "vertex_ai/gemini-live-2.5-flash", litellm_credential_name="cred_gone")
|
||||
]
|
||||
)
|
||||
passthrough_router = _passthrough_router(llm_router)
|
||||
monkeypatch.setattr(litellm, "credential_list", [])
|
||||
|
||||
assert passthrough_router.get_vertex_credentials_from_router_deployments(model=None) is None
|
||||
|
||||
|
||||
def _token_file(tmp_path, monkeypatch, value: str):
|
||||
monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path))
|
||||
token = tmp_path / "token"
|
||||
token.write_text(value)
|
||||
return token
|
||||
|
||||
|
||||
def test_env_oidc_file_reference_is_read_on_every_call(tmp_path, monkeypatch):
|
||||
token = _token_file(tmp_path, monkeypatch, "token-one")
|
||||
monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{token}")
|
||||
passthrough_router = _passthrough_router(None)
|
||||
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "token-one"
|
||||
|
||||
token.write_text("token-two")
|
||||
|
||||
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "token-two"
|
||||
|
||||
|
||||
def test_deployment_oidc_file_reference_is_resolved(tmp_path, monkeypatch):
|
||||
token = _token_file(tmp_path, monkeypatch, "deployment-token")
|
||||
llm_router = litellm.Router(model_list=[_flagged_deployment("openai/gpt-4o", api_key=f"oidc/file/{token}")])
|
||||
|
||||
assert (
|
||||
_passthrough_router(llm_router).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "deployment-token"
|
||||
)
|
||||
|
||||
|
||||
def test_oidc_file_reference_outside_allowed_dirs_is_refused(tmp_path, monkeypatch):
|
||||
allowed = tmp_path / "allowed"
|
||||
allowed.mkdir()
|
||||
monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(allowed))
|
||||
outside = tmp_path / "token"
|
||||
outside.write_text("not-allowed")
|
||||
monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{outside}")
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
|
||||
assert excinfo.value.status_code == 401
|
||||
assert f"oidc/file/{outside}" in str(excinfo.value.detail)
|
||||
assert "outside the allowed credential directories" in str(excinfo.value.detail)
|
||||
|
||||
|
||||
def test_missing_oidc_token_file_is_a_clear_auth_error(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS", str(tmp_path))
|
||||
monkeypatch.setenv("OPENAI_API_KEY", f"oidc/file/{tmp_path / 'missing-token'}")
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
|
||||
assert excinfo.value.status_code == 401
|
||||
assert f"oidc/file/{tmp_path / 'missing-token'}" in str(excinfo.value.detail)
|
||||
|
||||
|
||||
def test_oidc_env_reference_is_resolved(monkeypatch):
|
||||
monkeypatch.setenv("UPSTREAM_TOKEN", "env-token")
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oidc/env/UPSTREAM_TOKEN")
|
||||
|
||||
assert _passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None) == "env-token"
|
||||
|
||||
|
||||
def test_oidc_env_path_reference_is_not_resolved(tmp_path, monkeypatch):
|
||||
outside = tmp_path / "token"
|
||||
outside.write_text("must-not-be-read")
|
||||
monkeypatch.setenv("UPSTREAM_TOKEN_FILE", str(outside))
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "oidc/env_path/UPSTREAM_TOKEN_FILE")
|
||||
|
||||
assert (
|
||||
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "oidc/env_path/UPSTREAM_TOKEN_FILE"
|
||||
)
|
||||
|
||||
|
||||
def test_network_backed_oidc_reference_is_not_fetched_inline(monkeypatch):
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("network-backed oidc reference was resolved inside the request path")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.pass_through_endpoints.passthrough_endpoint_router.get_secret_str",
|
||||
lambda name, *a, **k: "oidc/google/https://example.com" if name == "OPENAI_API_KEY" else fail(),
|
||||
)
|
||||
|
||||
assert (
|
||||
_passthrough_router(None).get_credentials(custom_llm_provider="openai", region_name=None)
|
||||
== "oidc/google/https://example.com"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue