mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait
The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key.
This commit is contained in:
parent
e44c8de305
commit
1009886009
4 changed files with 113 additions and 22 deletions
|
|
@ -1,8 +1,10 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
@ -28,6 +30,12 @@ class VLLMError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
|
||||
if api_key is None:
|
||||
return {}
|
||||
return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"}
|
||||
|
||||
|
||||
class VLLMModelInfo(BaseLLMModelInfo):
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
@ -39,39 +47,51 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
if api_key is not None:
|
||||
headers["x-api-key"] = api_key
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
headers.update(_auth_headers(api_key))
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
api_base = api_base or get_secret_str("VLLM_API_BASE")
|
||||
if api_base is None:
|
||||
raise ValueError("VLLM_API_BASE is not set.")
|
||||
raise ValueError(
|
||||
"VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
|
||||
)
|
||||
return api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return None
|
||||
return api_key
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str | None:
|
||||
return model
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
api_base = VLLMModelInfo.get_api_base(api_base)
|
||||
endpoint: Final = "/v1/models"
|
||||
|
||||
url: Final = add_path_to_api_base(api_base, endpoint)
|
||||
headers: Final = {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} if api_key is not None else None
|
||||
response: Final = litellm.module_level_client.get(url=url, headers=headers)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
models: Final = response.json()["data"]
|
||||
|
||||
return [model["id"] for model in models]
|
||||
def get_models(
|
||||
self, api_key: str | None = None, api_base: str | None = None
|
||||
) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return []
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return VLLMError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
|
||||
class HostedVLLMModelInfo(VLLMModelInfo):
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
|
||||
if resolved_api_base is None:
|
||||
raise ValueError("HOSTED_VLLM_API_BASE is not set.")
|
||||
return resolved_api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT
|
||||
)
|
||||
response.raise_for_status()
|
||||
return [model["id"] for model in response.json()["data"]]
|
||||
|
|
|
|||
|
|
@ -10049,12 +10049,14 @@ class ProviderConfigManager:
|
|||
from litellm.llms.ollama.common_utils import OllamaModelInfo
|
||||
|
||||
return OllamaModelInfo()
|
||||
elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import (
|
||||
VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py
|
||||
)
|
||||
elif LlmProviders.VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import VLLMModelInfo
|
||||
|
||||
return VLLMModelInfo()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
|
||||
return HostedVLLMModelInfo()
|
||||
elif LlmProviders.LEMONADE == provider:
|
||||
return litellm.LemonadeChatConfig()
|
||||
elif LlmProviders.CLARIFAI == provider:
|
||||
|
|
|
|||
|
|
@ -51,3 +51,24 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m
|
|||
outbound: Final = json.loads(route.calls[0].request.content)
|
||||
assert outbound["model"] == "my-deployment"
|
||||
assert outbound["messages"] == [{"role": "user", "content": "Hello"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json={"ok": True})
|
||||
)
|
||||
response: Final = await allm_passthrough_route(
|
||||
method="POST",
|
||||
endpoint="v1/chat/completions",
|
||||
model="hosted_vllm/my-deployment",
|
||||
api_base=_API_BASE,
|
||||
api_key="deployment-key",
|
||||
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
client=AsyncHTTPHandler(),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
sent_headers: Final = route.calls[0].request.headers
|
||||
assert sent_headers["authorization"] == "Bearer deployment-key"
|
||||
assert sent_headers["x-api-key"] == "deployment-key"
|
||||
|
||||
|
|
|
|||
|
|
@ -481,6 +481,54 @@ def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(mon
|
|||
assert models_route.call_count == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1")
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key")
|
||||
monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000")
|
||||
passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]})
|
||||
)
|
||||
respx.get(
|
||||
"http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"}
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm")
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
assert not passthrough_server_route.called
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
|
||||
get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1")
|
||||
)
|
||||
|
||||
read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"]
|
||||
assert read_timeout is not None
|
||||
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
|
||||
|
||||
|
||||
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
|
||||
"""Regression test for #30358.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue