mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vllm): keep a router model's own key on the VLLM_API_BASE server and resolve an oidc VLLM_API_KEY
This commit is contained in:
parent
547013a7f3
commit
d573b95d33
3 changed files with 51 additions and 23 deletions
|
|
@ -13,7 +13,7 @@ from litellm.llms.openai_like.passthrough.transformation import (
|
|||
logged_text_completion_stream,
|
||||
relays_text_completion,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.secret_managers.main import UnresolvableSecretReference, get_secret_str, resolve_local_oidc_reference
|
||||
|
||||
from ..common_utils import VLLMModelInfo, endpoint_under_api_base
|
||||
|
||||
|
|
@ -29,11 +29,16 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
|
|||
return None if api_key == PLACEHOLDER_API_KEY else api_key
|
||||
|
||||
def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None:
|
||||
if api_base is None or api_base != get_secret_str("VLLM_API_BASE"):
|
||||
if api_key is not None or api_base != get_secret_str("VLLM_API_BASE"):
|
||||
return api_key
|
||||
if api_key is not None and api_key != get_secret_str("HOSTED_VLLM_API_KEY"):
|
||||
return api_key
|
||||
return get_secret_str("VLLM_API_KEY")
|
||||
vllm_api_key: Final = get_secret_str("VLLM_API_KEY")
|
||||
if vllm_api_key is None:
|
||||
return None
|
||||
match resolve_local_oidc_reference(vllm_api_key):
|
||||
case UnresolvableSecretReference():
|
||||
return None
|
||||
case str() as resolved:
|
||||
return resolved
|
||||
|
||||
def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]:
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
|
|
|||
|
|
@ -72,16 +72,17 @@ async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx
|
|||
assert "x-api-key" not in route.calls[0].request.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["v1/chat/completions", "chat/completions"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_route_hosted_vllm_reaches_v1_paths_under_an_api_base_ending_in_v1(
|
||||
respx_mock: respx.MockRouter,
|
||||
respx_mock: respx.MockRouter, endpoint: str
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json={"ok": True})
|
||||
)
|
||||
response: Final = await allm_passthrough_route(
|
||||
method="POST",
|
||||
endpoint="v1/chat/completions",
|
||||
endpoint=endpoint,
|
||||
model="hosted_vllm/my-deployment",
|
||||
api_base=f"{_API_BASE}/v1",
|
||||
api_key="sk-upstream",
|
||||
|
|
@ -114,18 +115,19 @@ async def test_allm_passthrough_route_hosted_vllm_sends_no_auth_for_a_keyless_de
|
|||
assert "x-api-key" not in route.calls[0].request.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hosted_vllm_api_key", ["sk-hosted-env", None])
|
||||
@pytest.mark.parametrize(
|
||||
("vllm_api_key", "expected_authorization"),
|
||||
[("sk-vllm-env", "Bearer sk-vllm-env"), ("oidc/env/VLLM_OIDC_TOKEN", "Bearer sk-vllm-oidc")],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vllm_api_base_key(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, hosted_vllm_api_key: str | None
|
||||
async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends_the_vllm_api_base_key(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str, expected_authorization: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
|
||||
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
|
||||
monkeypatch.setenv("VLLM_API_KEY", vllm_api_key)
|
||||
monkeypatch.setenv("VLLM_OIDC_TOKEN", "sk-vllm-oidc")
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
||||
if hosted_vllm_api_key is None:
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_KEY", hosted_vllm_api_key)
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
||||
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json={"ok": True})
|
||||
)
|
||||
|
|
@ -137,12 +139,25 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_sends_the_vll
|
|||
client=AsyncHTTPHandler(),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env"
|
||||
assert route.calls[0].request.headers["authorization"] == expected_authorization
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "api_key", "expected_authorization"),
|
||||
[
|
||||
(None, "sk-upstream", "Bearer sk-upstream"),
|
||||
(None, "sk-hosted-env", "Bearer sk-hosted-env"),
|
||||
(_API_BASE, "sk-hosted-env", "Bearer sk-hosted-env"),
|
||||
(None, None, "Bearer sk-hosted-env"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_deployment_api_key(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
async def test_allm_passthrough_route_hosted_vllm_on_the_vllm_api_base_sends_its_own_resolved_key(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
expected_authorization: str,
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
|
||||
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
|
||||
|
|
@ -155,9 +170,10 @@ async def test_allm_passthrough_route_hosted_vllm_without_api_base_keeps_the_dep
|
|||
method="POST",
|
||||
endpoint="v1/chat/completions",
|
||||
model="hosted_vllm/my-deployment",
|
||||
api_key="sk-upstream",
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
client=AsyncHTTPHandler(),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream"
|
||||
assert route.calls[0].request.headers["authorization"] == expected_authorization
|
||||
|
|
|
|||
|
|
@ -2757,9 +2757,16 @@ class TestVLLMProxyRoute:
|
|||
mock_factory_route.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vllm_api_base", ["http://vllm-upstream.test:8090", "http://vllm-upstream.test:8090/v1"])
|
||||
@pytest.mark.parametrize(
|
||||
("vllm_api_base", "request_path"),
|
||||
[
|
||||
("http://vllm-upstream.test:8090", "/vllm/v1/chat/completions"),
|
||||
("http://vllm-upstream.test:8090/v1", "/vllm/v1/chat/completions"),
|
||||
("http://vllm-upstream.test:8090/v1", "/vllm/chat/completions"),
|
||||
],
|
||||
)
|
||||
def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer(
|
||||
monkeypatch: pytest.MonkeyPatch, vllm_api_base: str
|
||||
monkeypatch: pytest.MonkeyPatch, vllm_api_base: str, request_path: str
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
|
|
@ -2777,7 +2784,7 @@ def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer(
|
|||
return_value=httpx.Response(200, json={"id": "chatcmpl-1"})
|
||||
)
|
||||
response: Final = TestClient(proxy_server.app).post(
|
||||
"/vllm/v1/chat/completions",
|
||||
request_path,
|
||||
json={"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue