mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-08 08:08:37 +00:00
Merge 740dadb0e9 into 60fc121ce8
This commit is contained in:
devin-ai-integration[bot]
GitHub
commit
cdfb9b0185
5 changed files with 335 additions and 31 deletions
|
|
@ -2,10 +2,12 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.utils import add_path_to_api_base
|
||||
|
|
@ -37,17 +39,31 @@ class VLLMError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
|
||||
|
||||
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
|
||||
if api_key is None or api_key == PLACEHOLDER_API_KEY:
|
||||
return {}
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
|
||||
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
|
||||
value: Final = litellm_params.get(key)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
class VLLMModelInfo(BaseLLMModelInfo):
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
upstream_key: Final = self.upstream_api_key(api_key, api_base)
|
||||
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
|
||||
|
||||
|
|
@ -58,7 +74,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
api_base = api_base or get_secret_str("VLLM_API_BASE")
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
|
|
@ -74,25 +90,55 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
def get_base_model(model: str) -> str | None:
|
||||
return model
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
api_base = VLLMModelInfo.get_api_base(api_base)
|
||||
api_key = VLLMModelInfo.get_api_key(api_key)
|
||||
endpoint: Final = "/v1/models"
|
||||
if api_base is None or api_key is None:
|
||||
raise ValueError(
|
||||
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
|
||||
)
|
||||
|
||||
url: Final = add_path_to_api_base(api_base, endpoint)
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=url,
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
models: Final = response.json()["data"]
|
||||
|
||||
return [model["id"] for model in models]
|
||||
def get_models(
|
||||
self, api_key: str | None = None, api_base: str | None = None
|
||||
) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return []
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return VLLMError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
|
||||
class HostedVLLMModelInfo(VLLMModelInfo):
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
|
||||
if resolved_api_base is None:
|
||||
raise ValueError("HOSTED_VLLM_API_BASE is not set.")
|
||||
return resolved_api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None)
|
||||
|
||||
def discover_models(
|
||||
self, litellm_params: Mapping[str, object] | None = None
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
params: Final = litellm_params or {}
|
||||
ssl_verify: Final = params.get("ssl_verify")
|
||||
return self._list_models(
|
||||
api_key=_str_param(params, "api_key"),
|
||||
api_base=_str_param(params, "api_base"),
|
||||
extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}),
|
||||
ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None,
|
||||
)
|
||||
|
||||
def _list_models(
|
||||
self,
|
||||
*,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
extra_headers: Mapping[str, str],
|
||||
ssl_verify: bool | str | None,
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
|
||||
response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get(
|
||||
url=url,
|
||||
headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))},
|
||||
timeout=NON_LLM_CONNECTION_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return [model["id"] for model in response.json()["data"]]
|
||||
|
|
|
|||
|
|
@ -67,9 +67,6 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
|
|||
) -> tuple["URL", str]:
|
||||
base_target_url: Final = self.get_api_base(api_base)
|
||||
|
||||
if base_target_url is None:
|
||||
raise Exception("VLLM api base not found")
|
||||
|
||||
return (
|
||||
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
|
||||
base_target_url,
|
||||
|
|
|
|||
|
|
@ -10505,12 +10505,14 @@ class ProviderConfigManager:
|
|||
from litellm.llms.ollama.common_utils import OllamaModelInfo
|
||||
|
||||
return OllamaModelInfo()
|
||||
elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import (
|
||||
VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py
|
||||
)
|
||||
elif LlmProviders.VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import VLLMModelInfo
|
||||
|
||||
return VLLMModelInfo()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
|
||||
return HostedVLLMModelInfo()
|
||||
elif LlmProviders.LEMONADE == provider:
|
||||
return litellm.LemonadeChatConfig()
|
||||
elif LlmProviders.CLARIFAI == provider:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,22 @@
|
|||
from collections.abc import Generator, Mapping
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import ssl
|
||||
import threading
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
from litellm import Router
|
||||
|
||||
|
|
@ -430,6 +441,237 @@ def test_wildcard_credential_hydration_preserves_deployment_params(
|
|||
}
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_lists_the_server_models_without_an_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get("http://vllm-without-key.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [
|
||||
{"id": "meta-llama/Llama-3.1-8B-Instruct"},
|
||||
{"id": "qwen2.5"},
|
||||
{"id": "openai/gpt-oss-20b"},
|
||||
]
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-without-key.test:8000/v1"),
|
||||
)
|
||||
|
||||
assert listed == [
|
||||
"hosted_vllm/meta-llama/Llama-3.1-8B-Instruct",
|
||||
"hosted_vllm/qwen2.5",
|
||||
"hosted_vllm/openai/gpt-oss-20b",
|
||||
]
|
||||
assert "authorization" not in models_route.calls.last.request.headers
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get(
|
||||
"http://vllm-with-key.test:8000/v1/models", headers={"Authorization": "Bearer vllm-test-key"}
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-with-key.test:8000/v1", api_key="vllm-test-key"),
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
assert models_route.call_count == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1")
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key")
|
||||
monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000")
|
||||
passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]})
|
||||
)
|
||||
respx.get(
|
||||
"http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"}
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm")
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
assert not passthrough_server_route.called
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
|
||||
get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1")
|
||||
)
|
||||
|
||||
read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"]
|
||||
assert read_timeout is not None
|
||||
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get(
|
||||
"http://vllm-behind-gateway.test:8000/v1/models",
|
||||
headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"},
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(
|
||||
model="hosted_vllm/*",
|
||||
api_base="http://vllm-behind-gateway.test:8000/v1",
|
||||
api_key="vllm-test-key",
|
||||
extra_headers={"X-Tenant": "team-a"},
|
||||
),
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
def _self_signed_cert(directory: Path) -> tuple[Path, Path]:
|
||||
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
|
||||
now: Final = datetime.datetime.now(datetime.UTC)
|
||||
cert: Final = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=1))
|
||||
.add_extension(
|
||||
x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False
|
||||
)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_path: Final = directory / "cert.pem"
|
||||
key_path: Final = directory / "key.pem"
|
||||
cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(
|
||||
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
|
||||
)
|
||||
return cert_path, key_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]:
|
||||
cert_path, key_path = _self_signed_cert(tmp_path)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
timeout = 5
|
||||
|
||||
def do_GET(self) -> None:
|
||||
body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # matches BaseHTTPRequestHandler
|
||||
pass
|
||||
|
||||
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert_path, key_path)
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield f"https://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify(
|
||||
monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.custom_httpx import http_handler
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setattr(litellm, "ssl_verify", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
|
||||
|
||||
verified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base)
|
||||
)
|
||||
unverified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False),
|
||||
)
|
||||
|
||||
assert verified == []
|
||||
assert unverified == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get("http://upstream-proxy.test:4000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]})
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"team-proxy/*",
|
||||
LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"),
|
||||
)
|
||||
|
||||
assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"]
|
||||
|
||||
|
||||
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
|
||||
"""Regression test for #30358.
|
||||
|
||||
|
|
|
|||
|
|
@ -9284,6 +9284,23 @@ def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_li
|
|||
assert (listings, catalog.call_count) == (([], []), 1)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_does_not_ask_a_timed_out_hosted_vllm_server_again_on_the_next_listing(
|
||||
reset_mock_cache: None,
|
||||
) -> None:
|
||||
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
|
||||
side_effect=httpx.ReadTimeout("vLLM did not answer")
|
||||
)
|
||||
deployment: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
|
||||
|
||||
listings: Final = tuple(
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=deployment)
|
||||
for _ in range(2)
|
||||
)
|
||||
|
||||
assert (listings, models_route.call_count) == (([], []), 1)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None:
|
||||
respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue