diff --git a/litellm/llms/vllm/passthrough/transformation.py b/litellm/llms/vllm/passthrough/transformation.py index 5f6c1ca74a0..2b253e7fa38 100644 --- a/litellm/llms/vllm/passthrough/transformation.py +++ b/litellm/llms/vllm/passthrough/transformation.py @@ -13,7 +13,12 @@ from litellm.llms.openai_like.passthrough.transformation import ( logged_text_completion_stream, relays_text_completion, ) -from litellm.secret_managers.main import UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference +from litellm.secret_managers.main import ( + GOOGLE_OIDC_SECRET_PREFIX, + UnresolvableSecretReference, + get_secret_str, + resolve_local_oidc_reference, +) from ..common_utils import VLLMModelInfo, endpoint_under_api_base @@ -29,10 +34,10 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): return None if api_key == PLACEHOLDER_API_KEY else api_key def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None: - if api_key is not None or api_base != get_secret_str("VLLM_API_BASE"): + if api_key is not None or not self._is_vllm_api_base(api_base): return api_key vllm_api_key: Final = get_secret_str("VLLM_API_KEY") - if vllm_api_key is None: + if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX): return None match resolve_local_oidc_reference(vllm_api_key): case UnresolvableSecretReference(): @@ -40,6 +45,11 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): case str() as resolved: return resolved + @staticmethod + def _is_vllm_api_base(api_base: str | None) -> bool: + vllm_api_base: Final = get_secret_str("VLLM_API_BASE") + return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/") + def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]: return {"Authorization": f"Bearer {api_key}"} diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index de3e27604a0..4997d736d0a 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -142,12 +142,39 @@ async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends assert route.calls[0].request.headers["authorization"] == expected_authorization +@pytest.mark.parametrize( + ("model_api_base", "vllm_api_base"), [(f"{_API_BASE}/", _API_BASE), (_API_BASE, f"{_API_BASE}/")] +) @pytest.mark.asyncio -async def test_allm_passthrough_route_empty_vllm_api_key_sends_no_authorization_header( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +async def test_allm_passthrough_route_keyless_hosted_vllm_matches_the_vllm_api_base_across_a_trailing_slash( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model_api_base: str, vllm_api_base: str +) -> None: + monkeypatch.setenv("VLLM_API_BASE", vllm_api_base) + monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env") + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(url__regex=rf"^{_API_BASE}/+v1/chat/completions$").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=model_api_base, + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env" + + +@pytest.mark.parametrize("vllm_api_key", ["", "oidc/google/audience"]) +@pytest.mark.asyncio +async def test_allm_passthrough_route_unusable_vllm_api_key_sends_no_authorization_header( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str ) -> None: monkeypatch.setenv("VLLM_API_BASE", _API_BASE) - monkeypatch.setenv("VLLM_API_KEY", "") + monkeypatch.setenv("VLLM_API_KEY", vllm_api_key) monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(