mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
fix(vllm): send the deployment api_key on /vllm passthrough calls
VLLMModelInfo.get_api_key always returned None, so /vllm/{endpoint} calls to a router
model reached the vLLM server with no auth header even when the deployment had an
api_key, and a server started with --api-key answered 401 (#23443).
Return the deployment key (or VLLM_API_KEY) and send it as Authorization: Bearer, the
header vllm serve --api-key checks. x-api-key was never actually sent before, since the
key was always None.
This commit is contained in:
parent
a3251a3bfd
commit
574d4b5670
2 changed files with 29 additions and 2 deletions
|
|
@ -40,7 +40,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
if api_key is not None:
|
||||
headers["x-api-key"] = api_key
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -54,7 +54,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return None
|
||||
return api_key or get_secret_str("VLLM_API_KEY")
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str | None:
|
||||
|
|
|
|||
|
|
@ -51,3 +51,30 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m
|
|||
outbound: Final = json.loads(route.calls[0].request.content)
|
||||
assert outbound["model"] == "my-deployment"
|
||||
assert outbound["messages"] == [{"role": "user", "content": "Hello"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json={"ok": True})
|
||||
)
|
||||
response: Final = await allm_passthrough_route(
|
||||
method="POST",
|
||||
endpoint="v1/chat/completions",
|
||||
model="hosted_vllm/my-deployment",
|
||||
api_base=_API_BASE,
|
||||
api_key="sk-upstream",
|
||||
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
client=AsyncHTTPHandler(),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream"
|
||||
assert "x-api-key" not in route.calls[0].request.headers
|
||||
|
||||
|
||||
def test_vllm_passthrough_api_key_falls_back_to_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("VLLM_API_KEY", "sk-from-env")
|
||||
assert VLLMPassthroughConfig.get_api_key(None) == "sk-from-env"
|
||||
assert VLLMPassthroughConfig.get_api_key("sk-explicit") == "sk-explicit"
|
||||
monkeypatch.delenv("VLLM_API_KEY")
|
||||
assert VLLMPassthroughConfig.get_api_key(None) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue