From d573b95d3301bf4607a85c687efa151fad13ac1e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:13:53 -0700 Subject: [PATCH] fix(vllm): keep a router model's own key on the VLLM_API_BASE server and resolve an oidc VLLM_API_KEY --- .../llms/vllm/passthrough/transformation.py | 15 ++++-- tests/unit/passthrough/test_main.py | 46 +++++++++++++------ .../test_llm_pass_through_endpoints.py | 13 ++++-- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/litellm/llms/vllm/passthrough/transformation.py b/litellm/llms/vllm/passthrough/transformation.py index 3d15f4a8c30..5f6c1ca74a0 100644 --- a/litellm/llms/vllm/passthrough/transformation.py +++ b/litellm/llms/vllm/passthrough/transformation.py @@ -13,7 +13,7 @@ from litellm.llms.openai_like.passthrough.transformation import ( logged_text_completion_stream, relays_text_completion, ) -from litellm.secret_managers.main import get_secret_str +from litellm.secret_managers.main import UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference from ..common_utils import VLLMModelInfo, endpoint_under_api_base @@ -29,11 +29,16 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): return None if api_key == PLACEHOLDER_API_KEY else api_key def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None: - if api_base is None or api_base != get_secret_str("VLLM_API_BASE"): + if api_key is not None or api_base != get_secret_str("VLLM_API_BASE"): return api_key - if api_key is not None and api_key != get_secret_str("HOSTED_VLLM_API_KEY"): - return api_key - return get_secret_str("VLLM_API_KEY") + vllm_api_key: Final = get_secret_str("VLLM_API_KEY") + if vllm_api_key is None: + return None + match resolve_local_oidc_reference(vllm_api_key): + case UnresolvableSecretReference(): + return None + case str() as resolved: + return resolved def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]: return {"Authorization": f"Bearer {api_key}"} diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index c3d5635a30f..8f145bc01a2 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -72,16 +72,17 @@ async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx assert "x-api-key" not in route.calls[0].request.headers +@pytest.mark.parametrize("endpoint", ["v1/chat/completions", "chat/completions"]) @pytest.mark.asyncio async def test_allm_passthrough_route_hosted_vllm_reaches_v1_paths_under_an_api_base_ending_in_v1( - respx_mock: respx.MockRouter, + respx_mock: respx.MockRouter, endpoint: str ) -> None: route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( return_value=httpx.Response(200, json={"ok": True}) ) response: Final = await allm_passthrough_route( method="POST", - endpoint="v1/chat/completions", + endpoint=endpoint, model="hosted_vllm/my-deployment", api_base=f"{_API_BASE}/v1", api_key="sk-upstream", @@ -114,18 +115,19 @@ async def test_allm_passthrough_route_hosted_vllm_sends_no_auth_for_a_keyless_de assert "x-api-key" not in route.calls[0].request.headers -@pytest.mark.parametrize("hosted_vllm_api_key", ["sk-hosted-env", None]) +@pytest.mark.parametrize( + ("vllm_api_key", "expected_authorization"), + [("sk-vllm-env", "Bearer sk-vllm-env"), ("oidc/env/VLLM_OIDC_TOKEN", "Bearer sk-vllm-oidc")], +) @pytest.mark.asyncio -async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vllm_api_base_key( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, hosted_vllm_api_key: str | None +async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends_the_vllm_api_base_key( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str, expected_authorization: str ) -> None: monkeypatch.setenv("VLLM_API_BASE", _API_BASE) - monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env") + monkeypatch.setenv("VLLM_API_KEY", vllm_api_key) + monkeypatch.setenv("VLLM_OIDC_TOKEN", "sk-vllm-oidc") monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) - if hosted_vllm_api_key is None: - monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) - else: - monkeypatch.setenv("HOSTED_VLLM_API_KEY", hosted_vllm_api_key) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( return_value=httpx.Response(200, json={"ok": True}) ) @@ -137,12 +139,25 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vll client=AsyncHTTPHandler(), ) assert response.status_code == 200 - assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env" + assert route.calls[0].request.headers["authorization"] == expected_authorization +@pytest.mark.parametrize( + ("api_base", "api_key", "expected_authorization"), + [ + (None, "sk-upstream", "Bearer sk-upstream"), + (None, "sk-hosted-env", "Bearer sk-hosted-env"), + (_API_BASE, "sk-hosted-env", "Bearer sk-hosted-env"), + (None, None, "Bearer sk-hosted-env"), + ], +) @pytest.mark.asyncio -async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_deployment_api_key( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +async def test_allm_passthrough_route_hosted_vllm_on_the_vllm_api_base_sends_its_own_resolved_key( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_base: str | None, + api_key: str | None, + expected_authorization: str, ) -> None: monkeypatch.setenv("VLLM_API_BASE", _API_BASE) monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env") @@ -155,9 +170,10 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_dep method="POST", endpoint="v1/chat/completions", model="hosted_vllm/my-deployment", - api_key="sk-upstream", + api_base=api_base, + api_key=api_key, json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, client=AsyncHTTPHandler(), ) assert response.status_code == 200 - assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream" + assert route.calls[0].request.headers["authorization"] == expected_authorization diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 4160fc3d8bf..5a5843062ef 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2757,9 +2757,16 @@ class TestVLLMProxyRoute: mock_factory_route.assert_awaited_once() -@pytest.mark.parametrize("vllm_api_base", ["http://vllm-upstream.test:8090", "http://vllm-upstream.test:8090/v1"]) +@pytest.mark.parametrize( + ("vllm_api_base", "request_path"), + [ + ("http://vllm-upstream.test:8090", "/vllm/v1/chat/completions"), + ("http://vllm-upstream.test:8090/v1", "/vllm/v1/chat/completions"), + ("http://vllm-upstream.test:8090/v1", "/vllm/chat/completions"), + ], +) def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer( - monkeypatch: pytest.MonkeyPatch, vllm_api_base: str + monkeypatch: pytest.MonkeyPatch, vllm_api_base: str, request_path: str ) -> None: from litellm.proxy import proxy_server @@ -2777,7 +2784,7 @@ def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer( return_value=httpx.Response(200, json={"id": "chatcmpl-1"}) ) response: Final = TestClient(proxy_server.app).post( - "/vllm/v1/chat/completions", + request_path, json={"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "hi"}]}, )