mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
d18efdfe13
* fix(vllm): send the deployment api_key on /vllm passthrough calls
VLLMModelInfo.get_api_key always returned None, so /vllm/{endpoint} calls to a router
model reached the vLLM server with no auth header even when the deployment had an
api_key, and a server started with --api-key answered 401 (#23443).
Return the deployment key (or VLLM_API_KEY) and send it as Authorization: Bearer, the
header vllm serve --api-key checks. x-api-key was never actually sent before, since the
key was always None.
* fix(vllm): send the pass-through key as a bearer on both /vllm routes
vllm serve --api-key only reads Authorization: Bearer, so a keyed vLLM
answered 401 on both pass-through routes. Send the key as a bearer and
keep x-api-key for gateways that read it. Router models send their own
deployment key and never fall back to VLLM_API_KEY, which belongs to
the /vllm route's own server.
* fix(vllm): send only the bearer on router models and stop doubling /v1 on /vllm
Router models now send the same Authorization: Bearer header /v1/chat/completions sends for that deployment, without the x-api-key copy that route never sent. The VLLM_API_BASE route keeps x-api-key next to the bearer, as it sent it before. A router model whose api_base already ends in /v1 no longer gets /v1/v1 on /vllm/v1 paths.
* fix(vllm): send no placeholder key from keyless router models and stop doubling /v1 on the VLLM_API_BASE route
* fix(vllm): send VLLM_API_KEY when a router model with no api_base is forwarded to VLLM_API_BASE
* fix(vllm): keep a router model's own key on the VLLM_API_BASE server and resolve an oidc VLLM_API_KEY
* fix(vllm): treat an empty key as no key so pass-through sends no empty bearer
* fix(vllm): send no bearer for a Google oidc key and match VLLM_API_BASE across a trailing slash
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
228 lines
9.1 KiB
Python
228 lines
9.1 KiB
Python
import json
|
|
from typing import Final
|
|
|
|
import httpx
|
|
import pytest
|
|
import respx
|
|
|
|
import litellm
|
|
|
|
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
|
from litellm.llms.vllm.passthrough.transformation import VLLMPassthroughConfig
|
|
from litellm.passthrough.main import allm_passthrough_route
|
|
from litellm.types.utils import LlmProviders
|
|
from litellm.utils import ProviderConfigManager
|
|
|
|
_API_BASE: Final = "http://vllm-upstream.test:8090"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
|
|
|
|
|
def test_hosted_vllm_resolves_vllm_passthrough_config() -> None:
|
|
cfg: Final = ProviderConfigManager.get_provider_passthrough_config(
|
|
model="hosted_vllm/my-deployment",
|
|
provider=LlmProviders.HOSTED_VLLM,
|
|
)
|
|
assert isinstance(cfg, VLLMPassthroughConfig)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_mock: respx.MockRouter) -> None:
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
client: Final = AsyncHTTPHandler()
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=_API_BASE,
|
|
json={
|
|
"model": "anything",
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
},
|
|
client=client,
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.call_count == 1
|
|
outbound: Final = json.loads(route.calls[0].request.content)
|
|
assert outbound["model"] == "my-deployment"
|
|
assert outbound["messages"] == [{"role": "user", "content": "Hello"}]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx_mock: respx.MockRouter) -> None:
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=_API_BASE,
|
|
api_key="sk-upstream",
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream"
|
|
assert "x-api-key" not in route.calls[0].request.headers
|
|
|
|
|
|
@pytest.mark.parametrize("endpoint", ["v1/chat/completions", "chat/completions"])
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_hosted_vllm_reaches_v1_paths_under_an_api_base_ending_in_v1(
|
|
respx_mock: respx.MockRouter, endpoint: str
|
|
) -> None:
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint=endpoint,
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=f"{_API_BASE}/v1",
|
|
api_key="sk-upstream",
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.call_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_hosted_vllm_sends_no_auth_for_a_keyless_deployment(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-route-only")
|
|
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=_API_BASE,
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert "authorization" not in route.calls[0].request.headers
|
|
assert "x-api-key" not in route.calls[0].request.headers
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("vllm_api_key", "expected_authorization"),
|
|
[("sk-vllm-env", "Bearer sk-vllm-env"), ("oidc/env/VLLM_OIDC_TOKEN", "Bearer sk-vllm-oidc")],
|
|
)
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends_the_vllm_api_base_key(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str, expected_authorization: str
|
|
) -> None:
|
|
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
|
|
monkeypatch.setenv("VLLM_API_KEY", vllm_api_key)
|
|
monkeypatch.setenv("VLLM_OIDC_TOKEN", "sk-vllm-oidc")
|
|
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
|
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.calls[0].request.headers["authorization"] == expected_authorization
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("model_api_base", "vllm_api_base"), [(f"{_API_BASE}/", _API_BASE), (_API_BASE, f"{_API_BASE}/")]
|
|
)
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_keyless_hosted_vllm_matches_the_vllm_api_base_across_a_trailing_slash(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model_api_base: str, vllm_api_base: str
|
|
) -> None:
|
|
monkeypatch.setenv("VLLM_API_BASE", vllm_api_base)
|
|
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
|
|
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
|
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
|
route: Final = respx_mock.post(url__regex=rf"^{_API_BASE}/+v1/chat/completions$").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=model_api_base,
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env"
|
|
|
|
|
|
@pytest.mark.parametrize("vllm_api_key", ["", "oidc/google/audience"])
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_unusable_vllm_api_key_sends_no_authorization_header(
|
|
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str
|
|
) -> None:
|
|
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
|
|
monkeypatch.setenv("VLLM_API_KEY", vllm_api_key)
|
|
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
|
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert "authorization" not in route.calls[0].request.headers
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("api_base", "api_key", "expected_authorization"),
|
|
[
|
|
(None, "sk-upstream", "Bearer sk-upstream"),
|
|
(None, "sk-hosted-env", "Bearer sk-hosted-env"),
|
|
(_API_BASE, "sk-hosted-env", "Bearer sk-hosted-env"),
|
|
(None, None, "Bearer sk-hosted-env"),
|
|
],
|
|
)
|
|
@pytest.mark.asyncio
|
|
async def test_allm_passthrough_route_hosted_vllm_on_the_vllm_api_base_sends_its_own_resolved_key(
|
|
respx_mock: respx.MockRouter,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
api_base: str | None,
|
|
api_key: str | None,
|
|
expected_authorization: str,
|
|
) -> None:
|
|
monkeypatch.setenv("VLLM_API_BASE", _API_BASE)
|
|
monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env")
|
|
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
|
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "sk-hosted-env")
|
|
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
|
return_value=httpx.Response(200, json={"ok": True})
|
|
)
|
|
response: Final = await allm_passthrough_route(
|
|
method="POST",
|
|
endpoint="v1/chat/completions",
|
|
model="hosted_vllm/my-deployment",
|
|
api_base=api_base,
|
|
api_key=api_key,
|
|
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
|
client=AsyncHTTPHandler(),
|
|
)
|
|
assert response.status_code == 200
|
|
assert route.calls[0].request.headers["authorization"] == expected_authorization
|