fix(vllm): keep a router model's own key on the VLLM_API_BASE server and resolve an oidc VLLM_API_KEY

This commit is contained in:
mateo-berri 2026-10-29 21:13:53 -07:00
parent 547013a7f3
commit d573b95d33
3 changed files with 51 additions and 23 deletions

View file

@ -13,7 +13,7 @@ from litellm.llms.openai_like.passthrough.transformation import (
logged_text_completion_stream,
relays_text_completion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.secret_managers.main import UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
@ -29,11 +29,16 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
return None if api_key == PLACEHOLDER_API_KEY else api_key
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
if api_base is None or api_base != get_secret_str("VLLM_API_BASE"):
if api_key is not None or api_base != get_secret_str("VLLM_API_BASE"):
return api_key
if api_key is not None and api_key != get_secret_str("HOSTED_VLLM_API_KEY"):
return api_key
return get_secret_str("VLLM_API_KEY")
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
if vllm_api_key is None:
return None
match resolve_local_oidc_reference(vllm_api_key):
case UnresolvableSecretReference():
return None
case str() as resolved:
return resolved
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
return {"Authorization": f"Bearer {api_key}"}

View file

@ -72,16 +72,17 @@ async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx
assert "x-api-key" not in route.calls[0].request.headers
@pytest.mark.parametrize("endpoint", ["v1/chat/completions", "chat/completions"])
@pytest.mark.asyncio
async def test_allm_passthrough_route_hosted_vllm_reaches_v1_paths_under_an_api_base_ending_in_v1(
respx_mock: respx.MockRouter,
respx_mock: respx.MockRouter, endpoint: str
) -> None:
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
return_value=httpx.Response(200, json={"ok": True})
)
response: Final = await allm_passthrough_route(
method="POST",
endpoint="v1/chat/completions",
endpoint=endpoint,
model="hosted_vllm/my-deployment",
api_base=f"{_API_BASE}/v1",
api_key="sk-upstream",
@ -114,18 +115,19 @@ async def test_allm_passthrough_route_hosted_vllm_sends_no_auth_for_a_keyless_de
assert "x-api-key" not in route.calls[0].request.headers
@pytest.mark.parametrize("hosted_vllm_api_key", ["sk-hosted-env", None])
@pytest.mark.parametrize(
("vllm_api_key", "expected_authorization"),
[("sk-vllm-env", "Bearer sk-vllm-env"), ("oidc/env/VLLM_OIDC_TOKEN", "Bearer sk-vllm-oidc")],
)
@pytest.mark.asyncio
async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vllm_api_base_key(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, hosted_vllm_api_key: str | None
async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends_the_vllm_api_base_key(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str, expected_authorization: str
) -> None:
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
monkeypatch.setenv("VLLM_API_KEY", vllm_api_key)
monkeypatch.setenv("VLLM_OIDC_TOKEN", "sk-vllm-oidc")
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
if hosted_vllm_api_key is None:
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
else:
monkeypatch.setenv("HOSTED_VLLM_API_KEY", hosted_vllm_api_key)
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
return_value=httpx.Response(200, json={"ok": True})
)
@ -137,12 +139,25 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vll
client=AsyncHTTPHandler(),
)
assert response.status_code == 200
assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env"
assert route.calls[0].request.headers["authorization"] == expected_authorization
@pytest.mark.parametrize(
("api_base", "api_key", "expected_authorization"),
[
(None, "sk-upstream", "Bearer sk-upstream"),
(None, "sk-hosted-env", "Bearer sk-hosted-env"),
(_API_BASE, "sk-hosted-env", "Bearer sk-hosted-env"),
(None, None, "Bearer sk-hosted-env"),
],
)
@pytest.mark.asyncio
async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_deployment_api_key(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
async def test_allm_passthrough_route_hosted_vllm_on_the_vllm_api_base_sends_its_own_resolved_key(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
api_base: str | None,
api_key: str | None,
expected_authorization: str,
) -> None:
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
@ -155,9 +170,10 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_dep
method="POST",
endpoint="v1/chat/completions",
model="hosted_vllm/my-deployment",
api_key="sk-upstream",
api_base=api_base,
api_key=api_key,
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
client=AsyncHTTPHandler(),
)
assert response.status_code == 200
assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream"
assert route.calls[0].request.headers["authorization"] == expected_authorization

View file

@ -2757,9 +2757,16 @@ class TestVLLMProxyRoute:
mock_factory_route.assert_awaited_once()
@pytest.mark.parametrize("vllm_api_base", ["http://vllm-upstream.test:8090", "http://vllm-upstream.test:8090/v1"])
@pytest.mark.parametrize(
("vllm_api_base", "request_path"),
[
("http://vllm-upstream.test:8090", "/vllm/v1/chat/completions"),
("http://vllm-upstream.test:8090/v1", "/vllm/v1/chat/completions"),
("http://vllm-upstream.test:8090/v1", "/vllm/chat/completions"),
],
)
def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer(
monkeypatch: pytest.MonkeyPatch, vllm_api_base: str
monkeypatch: pytest.MonkeyPatch, vllm_api_base: str, request_path: str
) -> None:
from litellm.proxy import proxy_server
@ -2777,7 +2784,7 @@ def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer(
return_value=httpx.Response(200, json={"id": "chatcmpl-1"})
)
response: Final = TestClient(proxy_server.app).post(
"/vllm/v1/chat/completions",
request_path,
json={"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "hi"}]},
)